Document optional library and increase docs coverage (#1162)

* Document optional library and increase docs coverage
* Fix doc merge on function decl merge

Author: TristonianJones

cel/decls.go

@@ -142,10 +142,7 @@ func Constant(name string, t *Type, v ref.Val) EnvOption {
 
 // Variable creates an instance of a variable declaration with a variable name and type.
 func Variable(name string, t *Type) EnvOption {
-	return func(e *Env) (*Env, error) {
-		e.variables = append(e.variables, decls.NewVariable(name, t))
-		return e, nil
-	}
+	return VariableWithDoc(name, t, "")
 }
 
 // VariableWithDoc creates an instance of a variable declaration with a variable name, type, and doc string.
@@ -201,14 +198,7 @@ func Function(name string, opts ...FunctionOpt) EnvOption {
 		if err != nil {
 			return nil, err
 		}
-		if existing, found := e.functions[fn.Name()]; found {
-			fn, err = existing.Merge(fn)
-			if err != nil {
-				return nil, err
-			}
-		}
-		e.functions[fn.Name()] = fn
-		return e, nil
+		return FunctionDecls(fn)(e)
 	}
 }
 

cel/env_test.go

@@ -378,11 +378,14 @@ func TestEnvToConfig(t *testing.T) {
 			name: "optional lib - alt last()",
 			opts: []EnvOption{
 				OptionalTypes(),
-				Function("last", MemberOverload("string_last", []*Type{StringType}, StringType)),
+				Function("last",
+					FunctionDocs(`return the last value in a list, or last character in a string`),
+					MemberOverload("string_last", []*Type{StringType}, StringType)),
 			},
 			want: env.NewConfig("optional lib - alt last()").
 				AddExtensions(env.NewExtension("optional", math.MaxUint32)).
-				AddFunctions(env.NewFunction("last",
+				AddFunctions(env.NewFunctionWithDoc("last",
+					`return the last value in a list, or last character in a string`,
 					env.NewMemberOverload("string_last", env.NewTypeDesc("string"), []*env.TypeDesc{}, env.NewTypeDesc("string")),
 				)),
 		},

cel/library.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"math"
 
+	"github.com/google/cel-go/common"
 	"github.com/google/cel-go/common/ast"
 	"github.com/google/cel-go/common/decls"
 	"github.com/google/cel-go/common/env"
@@ -421,16 +422,29 @@ func (lib *optionalLib) CompileOptions() []EnvOption {
 		Types(types.OptionalType),
 
 		// Configure the optMap and optFlatMap macros.
-		Macros(ReceiverMacro(optMapMacro, 2, optMap)),
+		Macros(ReceiverMacro(optMapMacro, 2, optMap,
+			MacroDocs(`perform computation on the value if present and return the result as an optional`),
+			MacroExamples(
+				common.MultilineDescription(
+					`// sub with the prefix 'dev.cel' or optional.none()`,
+					`request.auth.tokens.?sub.optMap(id, 'dev.cel.' + id)`),
+				`optional.none().optMap(i, i * 2) // optional.none()`))),
 
 		// Global and member functions for working with optional values.
 		Function(optionalOfFunc,
+			FunctionDocs(`create a new optional_type(T) with a value where any value is considered valid`),
 			Overload("optional_of", []*Type{paramTypeV}, optionalTypeV,
+				OverloadExamples(`optional.of(1) // optional(1)`),
 				UnaryBinding(func(value ref.Val) ref.Val {
 					return types.OptionalOf(value)
 				}))),
 		Function(optionalOfNonZeroValueFunc,
+			FunctionDocs(`create a new optional_type(T) with a value, if the value is not a zero or empty value`),
 			Overload("optional_ofNonZeroValue", []*Type{paramTypeV}, optionalTypeV,
+				OverloadExamples(
+					`optional.ofNonZeroValue(null) // optional.none()`,
+					`optional.ofNonZeroValue("") // optional.none()`,
+					`optional.ofNonZeroValue("hello") // optional.of('hello')`),
 				UnaryBinding(func(value ref.Val) ref.Val {
 					v, isZeroer := value.(traits.Zeroer)
 					if !isZeroer || !v.IsZeroValue() {
@@ -439,18 +453,26 @@ func (lib *optionalLib) CompileOptions() []EnvOption {
 					return types.OptionalNone
 				}))),
 		Function(optionalNoneFunc,
+			FunctionDocs(`singleton value representing an optional without a value`),
 			Overload("optional_none", []*Type{}, optionalTypeV,
+				OverloadExamples(`optional.none()`),
 				FunctionBinding(func(values ...ref.Val) ref.Val {
 					return types.OptionalNone
 				}))),
 		Function(valueFunc,
+			FunctionDocs(`obtain the value contained by the optional, error if optional.none()`),
 			MemberOverload("optional_value", []*Type{optionalTypeV}, paramTypeV,
+				OverloadExamples(
+					`optional.of(1).value() // 1`,
+					`optional.none().value() // error`),
 				UnaryBinding(func(value ref.Val) ref.Val {
 					opt := value.(*types.Optional)
 					return opt.GetValue()
 				}))),
 		Function(hasValueFunc,
+			FunctionDocs(`determine whether the optional contains a value`),
 			MemberOverload("optional_hasValue", []*Type{optionalTypeV}, BoolType,
+				OverloadExamples(`optional.of({1: 2}).hasValue() // true`),
 				UnaryBinding(func(value ref.Val) ref.Val {
 					opt := value.(*types.Optional)
 					return types.Bool(opt.HasValue())
@@ -459,21 +481,43 @@ func (lib *optionalLib) CompileOptions() []EnvOption {
 		// Implementation of 'or' and 'orValue' are special-cased to support short-circuiting in the
 		// evaluation chain.
 		Function("or",
-			MemberOverload("optional_or_optional", []*Type{optionalTypeV, optionalTypeV}, optionalTypeV)),
+			FunctionDocs(`chain optional expressions together, picking the first valued optional expression`),
+			MemberOverload("optional_or_optional", []*Type{optionalTypeV, optionalTypeV}, optionalTypeV,
+				OverloadExamples(
+					`optional.none().or(optional.of(1)) // optional.of(1)`,
+					common.MultilineDescription(
+						`// either a value from the first list, a value from the second, or optional.none()`,
+						`[1, 2, 3][?x].or([3, 4, 5][?y])`)))),
 		Function("orValue",
-			MemberOverload("optional_orValue_value", []*Type{optionalTypeV, paramTypeV}, paramTypeV)),
+			FunctionDocs(`chain optional expressions together picking the first valued optional or the default value`),
+			MemberOverload("optional_orValue_value", []*Type{optionalTypeV, paramTypeV}, paramTypeV,
+				OverloadExamples(
+					common.MultilineDescription(
+						`// pick the value for the given key if the key exists, otherwise return 'you'`,
+						`{'hello': 'world', 'goodbye': 'cruel world'}[?greeting].orValue('you')`)))),
 
 		// OptSelect is handled specially by the type-checker, so the receiver's field type is used to determine the
 		// optput type.
 		Function(operators.OptSelect,
-			Overload("select_optional_field", []*Type{DynType, StringType}, optionalTypeV)),
+			FunctionDocs(`if the field is present create an optional of the field value, otherwise return optional.none()`),
+			Overload("select_optional_field", []*Type{DynType, StringType}, optionalTypeV,
+				OverloadExamples(
+					`msg.?field // optional.of(field) if non-empty, otherwise optional.none()`,
+					`msg.?field.?nested_field // optional.of(nested_field) if both field and nested_field are non-empty.`))),
 
 		// OptIndex is handled mostly like any other indexing operation on a list or map, so the type-checker can use
 		// these signatures to determine type-agreement without any special handling.
 		Function(operators.OptIndex,
-			Overload("list_optindex_optional_int", []*Type{listTypeV, IntType}, optionalTypeV),
+			FunctionDocs(`if the index is present create an optional of the field value, otherwise return optional.none()`),
+			Overload("list_optindex_optional_int", []*Type{listTypeV, IntType}, optionalTypeV,
+				OverloadExamples(`[1, 2, 3][?x] // element value if x is in the list size, else optional.none()`)),
 			Overload("optional_list_optindex_optional_int", []*Type{OptionalType(listTypeV), IntType}, optionalTypeV),
-			Overload("map_optindex_optional_value", []*Type{mapTypeKV, paramTypeK}, optionalTypeV),
+			Overload("map_optindex_optional_value", []*Type{mapTypeKV, paramTypeK}, optionalTypeV,
+				OverloadExamples(
+					`map_value[?key] // value at the key if present, else optional.none()`,
+					common.MultilineDescription(
+						`// map key-value if index is a valid map key, else optional.none()`,
+						`{0: 2, 2: 4, 6: 8}[?index]`))),
 			Overload("optional_map_optindex_optional_value", []*Type{OptionalType(mapTypeKV), paramTypeK}, optionalTypeV)),
 
 		// Index overloads to accommodate using an optional value as the operand.
@@ -482,45 +526,62 @@ func (lib *optionalLib) CompileOptions() []EnvOption {
 			Overload("optional_map_index_value", []*Type{OptionalType(mapTypeKV), paramTypeK}, optionalTypeV)),
 	}
 	if lib.version >= 1 {
-		opts = append(opts, Macros(ReceiverMacro(optFlatMapMacro, 2, optFlatMap)))
+		opts = append(opts, Macros(ReceiverMacro(optFlatMapMacro, 2, optFlatMap,
+			MacroDocs(`perform computation on the value if present and produce an optional value within the computation`),
+			MacroExamples(
+				common.MultilineDescription(
+					`// m = {'key': {}}`,
+					`m.?key.optFlatMap(k, k.?subkey) // optional.none()`),
+				common.MultilineDescription(
+					`// m = {'key': {'subkey': 'value'}}`,
+					`m.?key.optFlatMap(k, k.?subkey) // optional.of('value')`),
+			))))
 	}
 
 	if lib.version >= 2 {
 		opts = append(opts, Function("last",
+			FunctionDocs(`return the last value in a list if present, otherwise optional.none()`),
 			MemberOverload("list_last", []*Type{listTypeV}, optionalTypeV,
+				OverloadExamples(
+					`[].last() // optional.none()`,
+					`[1, 2, 3].last() ? optional.of(3)`),
 				UnaryBinding(func(v ref.Val) ref.Val {
 					list := v.(traits.Lister)
-					sz := list.Size().Value().(int64)
-
-					if sz == 0 {
+					sz := list.Size().(types.Int)
+					if sz == types.IntZero {
 						return types.OptionalNone
 					}
-
 					return types.OptionalOf(list.Get(types.Int(sz - 1)))
 				}),
 			),
 		))
 
 		opts = append(opts, Function("first",
+			FunctionDocs(`return the first value in a list if present, otherwise optional.none()`),
 			MemberOverload("list_first", []*Type{listTypeV}, optionalTypeV,
+				OverloadExamples(
+					`[].first() // optional.none()`,
+					`[1, 2, 3].first() ? optional.of(1)`),
 				UnaryBinding(func(v ref.Val) ref.Val {
 					list := v.(traits.Lister)
-					sz := list.Size().Value().(int64)
-
-					if sz == 0 {
+					sz := list.Size().(types.Int)
+					if sz == types.IntZero {
 						return types.OptionalNone
 					}
-
 					return types.OptionalOf(list.Get(types.Int(0)))
 				}),
 			),
 		))
 
 		opts = append(opts, Function(optionalUnwrapFunc,
+			FunctionDocs(`convert a list of optional values to a list containing only value which are not optional.none()`),
 			Overload("optional_unwrap", []*Type{listOptionalTypeV}, listTypeV,
+				OverloadExamples(`optional.unwrap([optional.of(1), optional.none()]) // [1]`),
 				UnaryBinding(optUnwrap))))
 		opts = append(opts, Function(unwrapOptFunc,
+			FunctionDocs(`convert a list of optional values to a list containing only value which are not optional.none()`),
 			MemberOverload("optional_unwrapOpt", []*Type{listOptionalTypeV}, listTypeV,
+				OverloadExamples(`[optional.of(1), optional.none()].unwrapOpt() // [1]`),
 				UnaryBinding(optUnwrap))))
 	}
 

cel/macro_test.go

@@ -0,0 +1,90 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cel
+
+import (
+	"testing"
+
+	"github.com/google/cel-go/common"
+	"github.com/google/cel-go/common/ast"
+)
+
+func TestGlobalVarArgMacro(t *testing.T) {
+	noopExpander := func(meh MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *common.Error) {
+		return nil, nil
+	}
+	varArgMacro := GlobalVarArgMacro("varargs", noopExpander)
+	if varArgMacro.ArgCount() != 0 {
+		t.Errorf("ArgCount() got %d, wanted 0", varArgMacro.ArgCount())
+	}
+	if varArgMacro.Function() != "varargs" {
+		t.Errorf("Function() got %q, wanted 'varargs'", varArgMacro.Function())
+	}
+	if varArgMacro.MacroKey() != "varargs:*:false" {
+		t.Errorf("MacroKey() got %q, wanted 'varargs:*:false'", varArgMacro.MacroKey())
+	}
+	if varArgMacro.IsReceiverStyle() {
+		t.Errorf("IsReceiverStyle() got %t, wanted false", varArgMacro.IsReceiverStyle())
+	}
+}
+
+func TestReceiverVarArgMacro(t *testing.T) {
+	noopExpander := func(meh MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *common.Error) {
+		return nil, nil
+	}
+	varArgMacro := ReceiverVarArgMacro("varargs", noopExpander)
+	if varArgMacro.ArgCount() != 0 {
+		t.Errorf("ArgCount() got %d, wanted 0", varArgMacro.ArgCount())
+	}
+	if varArgMacro.Function() != "varargs" {
+		t.Errorf("Function() got %q, wanted 'varargs'", varArgMacro.Function())
+	}
+	if varArgMacro.MacroKey() != "varargs:*:true" {
+		t.Errorf("MacroKey() got %q, wanted 'varargs:*:true'", varArgMacro.MacroKey())
+	}
+	if !varArgMacro.IsReceiverStyle() {
+		t.Errorf("IsReceiverStyle() got %t, wanted true", varArgMacro.IsReceiverStyle())
+	}
+}
+
+func TestDocumentation(t *testing.T) {
+	noopExpander := func(meh MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *common.Error) {
+		return nil, nil
+	}
+	varArgMacro := ReceiverVarArgMacro("varargs", noopExpander,
+		MacroDocs(`convert variable argument lists to a list literal`),
+		MacroExamples(`fn.varargs(1,2,3) // fn([1, 2, 3])`))
+	doc, ok := varArgMacro.(common.Documentor)
+	if !ok {
+		t.Fatal("macro does not implement Documenter interface")
+	}
+	d := doc.Documentation()
+	if d.Kind != common.DocMacro {
+		t.Errorf("Documentation() got kind %v, wanted DocMacro", d.Kind)
+	}
+	if d.Name != varArgMacro.Function() {
+		t.Errorf("Documentation() got name %q, wanted %q", d.Name, varArgMacro.Function())
+	}
+	if d.Description != `convert variable argument lists to a list literal` {
+		t.Errorf("Documentation() got description %q, wanted %q", d.Description, `convert variable argument lists to a list literal`)
+	}
+	if len(d.Children) != 1 {
+		t.Fatalf("macro documentation children got: %d", len(d.Children))
+	}
+	if d.Children[0].Description != `fn.varargs(1,2,3) // fn([1, 2, 3])` {
+		t.Errorf("macro documentation Children[0] got %s, wanted %s", d.Children[0].Description,
+			`fn.varargs(1,2,3) // fn([1, 2, 3])`)
+	}
+}

cel/prompt.go

@@ -116,7 +116,7 @@ const (
 	defaultPersona = `You are a software engineer with expertise in networking and application security
 authoring boolean Common Expression Language (CEL) expressions to ensure firewall,
 networking, authentication, and data access is only permitted when all conditions
-are satisified.`
+are satisfied.`
 
 	defaultFormatRules = `Output your response as a CEL expression.
 

cel/prompt_test.go

@@ -22,13 +22,13 @@ import (
 	"github.com/google/cel-go/test"
 )
 
-//go:embed testdata/basic.prompt.md
+//go:embed testdata/basic.prompt.txt
 var wantBasicPrompt string
 
-//go:embed testdata/macros.prompt.md
+//go:embed testdata/macros.prompt.txt
 var wantMacrosPrompt string
 
-//go:embed testdata/standard_env.prompt.md
+//go:embed testdata/standard_env.prompt.txt
 var wantStandardEnvPrompt string
 
 func TestPromptTemplate(t *testing.T) {

cel/testdata/BUILD.bazel

@@ -19,5 +19,5 @@ genrule(
 
 filegroup(
     name = "prompts",
-    srcs = glob(["*.prompt.md"]),
+    srcs = glob(["*.prompt.txt"]),
 )

cel/testdata/basic.prompt.md renamed to cel/testdata/basic.prompt.txt

@@ -1,7 +1,7 @@
 You are a software engineer with expertise in networking and application security
 authoring boolean Common Expression Language (CEL) expressions to ensure firewall,
 networking, authentication, and data access is only permitted when all conditions
-are satisified.
+are satisfied.
 
 Output your response as a CEL expression.
 

cel/testdata/macros.prompt.md renamed to cel/testdata/macros.prompt.txt

@@ -1,7 +1,7 @@
 You are a software engineer with expertise in networking and application security
 authoring boolean Common Expression Language (CEL) expressions to ensure firewall,
 networking, authentication, and data access is only permitted when all conditions
-are satisified.
+are satisfied.
 
 Output your response as a CEL expression.
 

cel/testdata/standard_env.prompt.md renamed to cel/testdata/standard_env.prompt.txt

@@ -1,7 +1,7 @@
 You are a software engineer with expertise in networking and application security
 authoring boolean Common Expression Language (CEL) expressions to ensure firewall,
 networking, authentication, and data access is only permitted when all conditions
-are satisified.
+are satisfied.
 
 Output your response as a CEL expression.