fix(go): display clear error message when logger fails (#3577)

Author: huangjeff5

go/plugins/googlecloud/googlecloud.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -45,6 +46,9 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
 )
 
+// Global sync.Once for showing logging setup instructions only once across all recovery attempts
+var showLoggingInstructionsOnce sync.Once
+
 // EnableGoogleCloudTelemetry enables comprehensive telemetry export to Google Cloud Observability suite.
 // This directly initializes telemetry without requiring plugin registration.
 //
@@ -222,6 +226,10 @@ func FlushMetrics(ctx context.Context) error {
 }
 
 func setLogHandler(projectID string, level slog.Leveler, credentials *google.Credentials) error {
+	return setupGCPLogger(projectID, level, credentials)
+}
+
+func setupGCPLogger(projectID string, level slog.Leveler, credentials *google.Credentials) error {
 	var clientOpts []option.ClientOption
 	if credentials != nil {
 		clientOpts = append(clientOpts, option.WithCredentials(credentials))
@@ -231,6 +239,28 @@ func setLogHandler(projectID string, level slog.Leveler, credentials *google.Cre
 	if err != nil {
 		return fmt.Errorf("failed to create logging client: %w", err)
 	}
+	// Set up error handling for async logging failures with recursive recovery
+	c.OnError = func(err error) {
+		fallbackHandler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+			Level: level,
+		})
+		slog.SetDefault(slog.New(fallbackHandler))
+		slog.Warn("Switched to stderr logging due to Google Cloud logging failure", "error", err)
+		slog.Error("Unable to send logs to Google Cloud", "error", err)
+		if loggingDenied(err) {
+			showLoggingInstructionsOnce.Do(func() {
+				fmt.Fprint(os.Stderr, loggingDeniedHelpText(projectID))
+			})
+		}
+
+		// Assume the logger is compromised, and we need a new one
+		// Reinitialize the logger with a new instance with the same config
+		if setupErr := setupGCPLogger(projectID, level, credentials); setupErr == nil {
+			slog.Info("Initialized a new GcpLogger")
+		} else {
+			slog.Error("Failed to reinitialize GCP logger", "error", setupErr)
+		}
+	}
 	logger := c.Logger("genkit_log")
 	slog.SetDefault(slog.New(newHandler(level, logger.Log, projectID)))
 	return nil

go/plugins/googlecloud/slog_handler.go

@@ -22,9 +22,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
-	"os"
 	"strings"
-	"time"
 
 	"cloud.google.com/go/logging"
 	"github.com/jba/slog/withsupport"
@@ -37,29 +35,21 @@ const MetadataKey = "metadata"
 
 // Enhanced handler with error handling
 type handler struct {
-	level              slog.Leveler
-	handleEntry        func(logging.Entry)
-	goa                *withsupport.GroupOrAttrs
-	projectID          string
-	fallbackHandler    slog.Handler
-	instructionsLogged bool
+	level       slog.Leveler
+	handleEntry func(logging.Entry)
+	goa         *withsupport.GroupOrAttrs
+	projectID   string
 }
 
 func newHandler(level slog.Leveler, f func(logging.Entry), projectID string) *handler {
 	if level == nil {
 		level = slog.LevelInfo
 	}
 
-	// Create fallback handler for when GCP logging fails
-	fallbackHandler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
-		Level: level,
-	})
-
 	return &handler{
-		level:           level,
-		handleEntry:     f,
-		projectID:       projectID,
-		fallbackHandler: fallbackHandler,
+		level:       level,
+		handleEntry: f,
+		projectID:   projectID,
 	}
 }
 
@@ -69,21 +59,19 @@ func (h *handler) Enabled(ctx context.Context, level slog.Level) bool {
 
 func (h *handler) WithAttrs(as []slog.Attr) slog.Handler {
 	return &handler{
-		level:           h.level,
-		handleEntry:     h.handleEntry,
-		goa:             h.goa.WithAttrs(as),
-		projectID:       h.projectID,
-		fallbackHandler: h.fallbackHandler,
+		level:       h.level,
+		handleEntry: h.handleEntry,
+		goa:         h.goa.WithAttrs(as),
+		projectID:   h.projectID,
 	}
 }
 
 func (h *handler) WithGroup(name string) slog.Handler {
 	return &handler{
-		level:           h.level,
-		handleEntry:     h.handleEntry,
-		goa:             h.goa.WithGroup(name),
-		projectID:       h.projectID,
-		fallbackHandler: h.fallbackHandler,
+		level:       h.level,
+		handleEntry: h.handleEntry,
+		goa:         h.goa.WithGroup(name),
+		projectID:   h.projectID,
 	}
 }
 
@@ -96,95 +84,16 @@ func (h *handler) Handle(ctx context.Context, r slog.Record) error {
 		strings.Contains(message, "google.logging.v2.LoggingServiceV2")
 
 	if isInternalGoogleCloudLog {
-		// Skip internal Google Cloud SDK logs, but send to fallback for debugging if needed
-		return h.fallbackHandler.Handle(ctx, r)
+		// Skip these logs - they're noise
+		return nil
 	}
 
 	entry := h.recordToEntry(ctx, r)
 
-	// Try to send to GCP with error handling and recovery
-	if err := h.handleWithRecovery(entry); err != nil {
-		// Fall back to local logging if GCP fails
-		return h.fallbackHandler.Handle(ctx, r)
-	}
-
+	h.handleEntry(entry)
 	return nil
 }
 
-// handleWithRecovery attempts to send the log entry to GCP
-func (h *handler) handleWithRecovery(entry logging.Entry) error {
-	// Attempt to send the log entry
-	defer func() {
-		if r := recover(); r != nil {
-			h.handleError(fmt.Errorf("panic in GCP logging: %v", r))
-		}
-	}()
-
-	// Create a channel to capture any errors from the async logging operation
-	errChan := make(chan error, 1)
-
-	// Wrap the handleEntry function to capture errors
-	wrappedHandleEntry := func(entry logging.Entry) {
-		defer func() {
-			if r := recover(); r != nil {
-				errChan <- fmt.Errorf("logging operation panic: %v", r)
-			} else {
-				errChan <- nil
-			}
-		}()
-		h.handleEntry(entry)
-	}
-
-	go wrappedHandleEntry(entry)
-
-	// Wait for completion with timeout
-	select {
-	case err := <-errChan:
-		if err != nil {
-			h.handleError(err)
-			return err
-		}
-		return nil
-	case <-time.After(5 * time.Second):
-		err := fmt.Errorf("GCP logging timeout")
-		h.handleError(err)
-		return err
-	}
-}
-
-// handleError processes logging errors and triggers immediate recovery
-func (h *handler) handleError(err error) {
-	// Check if this is a permission denied error for helpful messaging
-	if loggingDenied(err) {
-		h.logPermissionError(err)
-	} else {
-		// Log generic error
-		h.fallbackHandler.Handle(context.Background(), slog.NewRecord(
-			time.Now(),
-			slog.LevelError,
-			fmt.Sprintf("Unable to send logs to Google Cloud: %v", err),
-			0,
-		))
-	}
-
-}
-
-// logPermissionError logs helpful permission error messages (only once)
-func (h *handler) logPermissionError(err error) {
-	if !h.instructionsLogged {
-		h.instructionsLogged = true
-		helpText := loggingDeniedHelpText(h.projectID)
-		errorMsg := fmt.Sprintf("Unable to send logs to Google Cloud: %v\n\n%s\n", err, helpText)
-
-		h.fallbackHandler.Handle(context.Background(), slog.NewRecord(
-			time.Now(),
-			slog.LevelError,
-			errorMsg,
-			0,
-		))
-	}
-}
-
 func (h *handler) recordToEntry(ctx context.Context, r slog.Record) logging.Entry {
 	span := trace.SpanFromContext(ctx)
 

go/plugins/googlecloud/utils.go

@@ -121,7 +121,9 @@ gcloud projects add-iam-policy-binding %s \
     --member=serviceAccount:${SERVICE_ACCOUNT_EMAIL} \
     --role=%s
 
-For more information, see: https://cloud.google.com/docs/authentication/getting-started`, role, projectID, role)
+For more information, see: https://cloud.google.com/docs/authentication/getting-started
+
+`, role, projectID, role)
 }
 
 // loggingDeniedHelpText provides specific help for logging permission errors