Skip to content

Commit 2e630c7

Browse files
yoshi-automationsofisl
yoshi-automation
authored and
sofisl
committed
feat(dataflow): update the API
#### dataflow:v1b3 The following keys were added: - schemas.BoundedTrie.description - schemas.BoundedTrie.id - schemas.BoundedTrie.properties.bound.description - schemas.BoundedTrie.properties.bound.format - schemas.BoundedTrie.properties.bound.type - schemas.BoundedTrie.properties.root.$ref - schemas.BoundedTrie.properties.root.description - schemas.BoundedTrie.properties.singleton.description - schemas.BoundedTrie.properties.singleton.items.type - schemas.BoundedTrie.properties.singleton.type - schemas.BoundedTrie.type - schemas.BoundedTrieNode.description - schemas.BoundedTrieNode.id - schemas.BoundedTrieNode.properties.children.additionalProperties.$ref - schemas.BoundedTrieNode.properties.children.description - schemas.BoundedTrieNode.properties.children.type - schemas.BoundedTrieNode.properties.truncated.description - schemas.BoundedTrieNode.properties.truncated.type - schemas.BoundedTrieNode.type - schemas.CounterUpdate.properties.boundedTrie.$ref - schemas.CounterUpdate.properties.boundedTrie.description - schemas.DataflowGaugeValue.description - schemas.DataflowGaugeValue.id - schemas.DataflowGaugeValue.properties.measuredTime.description - schemas.DataflowGaugeValue.properties.measuredTime.format - schemas.DataflowGaugeValue.properties.measuredTime.type - schemas.DataflowGaugeValue.properties.value.description - schemas.DataflowGaugeValue.properties.value.format - schemas.DataflowGaugeValue.properties.value.type - schemas.DataflowGaugeValue.type - schemas.GPUUsage.description - schemas.GPUUsage.id - schemas.GPUUsage.properties.timestamp.description - schemas.GPUUsage.properties.timestamp.format - schemas.GPUUsage.properties.timestamp.type - schemas.GPUUsage.properties.utilization.$ref - schemas.GPUUsage.properties.utilization.description - schemas.GPUUsage.type - schemas.GPUUtilization.description - schemas.GPUUtilization.id - schemas.GPUUtilization.properties.rate.description - schemas.GPUUtilization.properties.rate.format - schemas.GPUUtilization.properties.rate.type - schemas.GPUUtilization.type - schemas.MetricUpdate.properties.trie.description - schemas.MetricUpdate.properties.trie.type - schemas.MetricValue.properties.valueGauge64.$ref - schemas.MetricValue.properties.valueGauge64.description - schemas.ResourceUtilizationReport.properties.gpuUsage.description - schemas.ResourceUtilizationReport.properties.gpuUsage.items.$ref - schemas.ResourceUtilizationReport.properties.gpuUsage.type The following keys were changed: - schemas.CounterUpdate.description - schemas.FlexTemplateRuntimeEnvironment.description - schemas.MetricUpdate.description - schemas.MetricUpdate.properties.set.description - schemas.RuntimeEnvironment.description
1 parent 4eedeb4 commit 2e630c7

File tree

2 files changed

+197
-11
lines changed

2 files changed

+197
-11
lines changed

‎discovery/dataflow-v1b3.json

Lines changed: 111 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2208,7 +2208,7 @@
22082208
}
22092209
}
22102210
},
2211-
"revision": "20240817",
2211+
"revision": "20250106",
22122212
"rootUrl": "https://dataflow.googleapis.com/",
22132213
"schemas": {
22142214
"ApproximateProgress": {
@@ -2413,6 +2413,47 @@
24132413
},
24142414
"type": "object"
24152415
},
2416+
"BoundedTrie": {
2417+
"description": "The message type used for encoding metrics of type bounded trie.",
2418+
"id": "BoundedTrie",
2419+
"properties": {
2420+
"bound": {
2421+
"description": "The maximum number of elements to store before truncation.",
2422+
"format": "int32",
2423+
"type": "integer"
2424+
},
2425+
"root": {
2426+
"$ref": "BoundedTrieNode",
2427+
"description": "A compact representation of all the elements in this trie."
2428+
},
2429+
"singleton": {
2430+
"description": "A more efficient representation for metrics consisting of a single value.",
2431+
"items": {
2432+
"type": "string"
2433+
},
2434+
"type": "array"
2435+
}
2436+
},
2437+
"type": "object"
2438+
},
2439+
"BoundedTrieNode": {
2440+
"description": "A single node in a BoundedTrie.",
2441+
"id": "BoundedTrieNode",
2442+
"properties": {
2443+
"children": {
2444+
"additionalProperties": {
2445+
"$ref": "BoundedTrieNode"
2446+
},
2447+
"description": "Children of this node. Must be empty if truncated is true.",
2448+
"type": "object"
2449+
},
2450+
"truncated": {
2451+
"description": "Whether this node has been truncated. A truncated leaf represents possibly many children with the same prefix.",
2452+
"type": "boolean"
2453+
}
2454+
},
2455+
"type": "object"
2456+
},
24162457
"BucketOptions": {
24172458
"description": "`BucketOptions` describes the bucket boundaries used in the histogram.",
24182459
"id": "BucketOptions",
@@ -2731,13 +2772,17 @@
27312772
"type": "object"
27322773
},
27332774
"CounterUpdate": {
2734-
"description": "An update to a Counter sent from a worker.",
2775+
"description": "An update to a Counter sent from a worker. Next ID: 17",
27352776
"id": "CounterUpdate",
27362777
"properties": {
27372778
"boolean": {
27382779
"description": "Boolean value for And, Or.",
27392780
"type": "boolean"
27402781
},
2782+
"boundedTrie": {
2783+
"$ref": "BoundedTrie",
2784+
"description": "Bounded trie data"
2785+
},
27412786
"cumulative": {
27422787
"description": "True if this counter is reported as the total cumulative aggregate value accumulated since the worker started working on this WorkItem. By default this is false, indicating that this counter is reported as a delta.",
27432788
"type": "boolean"
@@ -2926,6 +2971,23 @@
29262971
},
29272972
"type": "object"
29282973
},
2974+
"DataflowGaugeValue": {
2975+
"description": "The gauge value of a metric.",
2976+
"id": "DataflowGaugeValue",
2977+
"properties": {
2978+
"measuredTime": {
2979+
"description": "The timestamp when the gauge was recorded.",
2980+
"format": "google-datetime",
2981+
"type": "string"
2982+
},
2983+
"value": {
2984+
"description": "The value of the gauge.",
2985+
"format": "int64",
2986+
"type": "string"
2987+
}
2988+
},
2989+
"type": "object"
2990+
},
29292991
"DataflowHistogramValue": {
29302992
"description": "Summary statistics for a population of values. HistogramValue contains a sequence of buckets and gives a count of values that fall into each bucket. Bucket boundares are defined by a formula and bucket widths are either fixed or exponentially increasing.",
29312993
"id": "DataflowHistogramValue",
@@ -3446,7 +3508,7 @@
34463508
"type": "object"
34473509
},
34483510
"FlexTemplateRuntimeEnvironment": {
3449-
"description": "The environment values to be set at runtime for flex template. LINT.IfChange",
3511+
"description": "The environment values to be set at runtime for flex template.",
34503512
"id": "FlexTemplateRuntimeEnvironment",
34513513
"properties": {
34523514
"additionalExperiments": {
@@ -3632,6 +3694,34 @@
36323694
},
36333695
"type": "object"
36343696
},
3697+
"GPUUsage": {
3698+
"description": "Information about the GPU usage on the worker.",
3699+
"id": "GPUUsage",
3700+
"properties": {
3701+
"timestamp": {
3702+
"description": "Required. Timestamp of the measurement.",
3703+
"format": "google-datetime",
3704+
"type": "string"
3705+
},
3706+
"utilization": {
3707+
"$ref": "GPUUtilization",
3708+
"description": "Required. Utilization info about the GPU."
3709+
}
3710+
},
3711+
"type": "object"
3712+
},
3713+
"GPUUtilization": {
3714+
"description": "Utilization details about the GPU.",
3715+
"id": "GPUUtilization",
3716+
"properties": {
3717+
"rate": {
3718+
"description": "Required. GPU utilization rate of any kernel over the last sample period in the range of [0, 1].",
3719+
"format": "double",
3720+
"type": "number"
3721+
}
3722+
},
3723+
"type": "object"
3724+
},
36353725
"GetDebugConfigRequest": {
36363726
"description": "Request to get updated debug configuration for component.",
36373727
"id": "GetDebugConfigRequest",
@@ -4663,7 +4753,7 @@
46634753
"type": "object"
46644754
},
46654755
"MetricUpdate": {
4666-
"description": "Describes the state of a metric.",
4756+
"description": "Describes the state of a metric. Next ID: 14",
46674757
"id": "MetricUpdate",
46684758
"properties": {
46694759
"cumulative": {
@@ -4703,7 +4793,11 @@
47034793
"type": "any"
47044794
},
47054795
"set": {
4706-
"description": "Worker-computed aggregate value for the \"Set\" aggregation kind. The only possible value type is a list of Values whose type can be Long, Double, or String, according to the metric's type. All Values in the list must be of the same type.",
4796+
"description": "Worker-computed aggregate value for the \"Set\" aggregation kind. The only possible value type is a list of Values whose type can be Long, Double, String, or BoundedTrie according to the metric's type. All Values in the list must be of the same type.",
4797+
"type": "any"
4798+
},
4799+
"trie": {
4800+
"description": "Worker-computed aggregate value for the \"Trie\" aggregation kind. The only possible value type is a BoundedTrieNode.",
47074801
"type": "any"
47084802
},
47094803
"updateTime": {
@@ -4729,6 +4823,10 @@
47294823
"description": "Optional. Set of metric labels for this metric.",
47304824
"type": "object"
47314825
},
4826+
"valueGauge64": {
4827+
"$ref": "DataflowGaugeValue",
4828+
"description": "Non-cumulative int64 value of this metric."
4829+
},
47324830
"valueHistogram": {
47334831
"$ref": "DataflowHistogramValue",
47344832
"description": "Histogram value of this metric."
@@ -5468,6 +5566,13 @@
54685566
},
54695567
"type": "array"
54705568
},
5569+
"gpuUsage": {
5570+
"description": "Optional. GPU usage samples.",
5571+
"items": {
5572+
"$ref": "GPUUsage"
5573+
},
5574+
"type": "array"
5575+
},
54715576
"memoryInfo": {
54725577
"description": "Memory utilization samples.",
54735578
"items": {
@@ -5485,7 +5590,7 @@
54855590
"type": "object"
54865591
},
54875592
"RuntimeEnvironment": {
5488-
"description": "The environment values to set at runtime. LINT.IfChange",
5593+
"description": "The environment values to set at runtime.",
54895594
"id": "RuntimeEnvironment",
54905595
"properties": {
54915596
"additionalExperiments": {

‎src/apis/dataflow/v1b3.ts

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,36 @@ export namespace dataflow_v1b3 {
272272
*/
273273
tableId?: string | null;
274274
}
275+
/**
276+
* The message type used for encoding metrics of type bounded trie.
277+
*/
278+
export interface Schema$BoundedTrie {
279+
/**
280+
* The maximum number of elements to store before truncation.
281+
*/
282+
bound?: number | null;
283+
/**
284+
* A compact representation of all the elements in this trie.
285+
*/
286+
root?: Schema$BoundedTrieNode;
287+
/**
288+
* A more efficient representation for metrics consisting of a single value.
289+
*/
290+
singleton?: string[] | null;
291+
}
292+
/**
293+
* A single node in a BoundedTrie.
294+
*/
295+
export interface Schema$BoundedTrieNode {
296+
/**
297+
* Children of this node. Must be empty if truncated is true.
298+
*/
299+
children?: {[key: string]: Schema$BoundedTrieNode} | null;
300+
/**
301+
* Whether this node has been truncated. A truncated leaf represents possibly many children with the same prefix.
302+
*/
303+
truncated?: boolean | null;
304+
}
275305
/**
276306
* `BucketOptions` describes the bucket boundaries used in the histogram.
277307
*/
@@ -474,13 +504,17 @@ export namespace dataflow_v1b3 {
474504
name?: Schema$CounterStructuredName;
475505
}
476506
/**
477-
* An update to a Counter sent from a worker.
507+
* An update to a Counter sent from a worker. Next ID: 17
478508
*/
479509
export interface Schema$CounterUpdate {
480510
/**
481511
* Boolean value for And, Or.
482512
*/
483513
boolean?: boolean | null;
514+
/**
515+
* Bounded trie data
516+
*/
517+
boundedTrie?: Schema$BoundedTrie;
484518
/**
485519
* True if this counter is reported as the total cumulative aggregate value accumulated since the worker started working on this WorkItem. By default this is false, indicating that this counter is reported as a delta.
486520
*/
@@ -602,6 +636,19 @@ export namespace dataflow_v1b3 {
602636
*/
603637
vmInstance?: string | null;
604638
}
639+
/**
640+
* The gauge value of a metric.
641+
*/
642+
export interface Schema$DataflowGaugeValue {
643+
/**
644+
* The timestamp when the gauge was recorded.
645+
*/
646+
measuredTime?: string | null;
647+
/**
648+
* The value of the gauge.
649+
*/
650+
value?: string | null;
651+
}
605652
/**
606653
* Summary statistics for a population of values. HistogramValue contains a sequence of buckets and gives a count of values that fall into each bucket. Bucket boundares are defined by a formula and bucket widths are either fixed or exponentially increasing.
607654
*/
@@ -983,7 +1030,7 @@ export namespace dataflow_v1b3 {
9831030
inputs?: Schema$InstructionInput[];
9841031
}
9851032
/**
986-
* The environment values to be set at runtime for flex template. LINT.IfChange
1033+
* The environment values to be set at runtime for flex template.
9871034
*/
9881035
export interface Schema$FlexTemplateRuntimeEnvironment {
9891036
/**
@@ -1156,6 +1203,28 @@ export namespace dataflow_v1b3 {
11561203
*/
11571204
templateType?: string | null;
11581205
}
1206+
/**
1207+
* Information about the GPU usage on the worker.
1208+
*/
1209+
export interface Schema$GPUUsage {
1210+
/**
1211+
* Required. Timestamp of the measurement.
1212+
*/
1213+
timestamp?: string | null;
1214+
/**
1215+
* Required. Utilization info about the GPU.
1216+
*/
1217+
utilization?: Schema$GPUUtilization;
1218+
}
1219+
/**
1220+
* Utilization details about the GPU.
1221+
*/
1222+
export interface Schema$GPUUtilization {
1223+
/**
1224+
* Required. GPU utilization rate of any kernel over the last sample period in the range of [0, 1].
1225+
*/
1226+
rate?: number | null;
1227+
}
11591228
/**
11601229
* Histogram of value counts for a distribution. Buckets have an inclusive lower bound and exclusive upper bound and use "1,2,5 bucketing": The first bucket range is from [0,1) and all subsequent bucket boundaries are powers of ten multiplied by 1, 2, or 5. Thus, bucket boundaries are 0, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, ... Negative values are not supported.
11611230
*/
@@ -1830,7 +1899,7 @@ export namespace dataflow_v1b3 {
18301899
origin?: string | null;
18311900
}
18321901
/**
1833-
* Describes the state of a metric.
1902+
* Describes the state of a metric. Next ID: 14
18341903
*/
18351904
export interface Schema$MetricUpdate {
18361905
/**
@@ -1870,9 +1939,13 @@ export namespace dataflow_v1b3 {
18701939
*/
18711940
scalar?: any | null;
18721941
/**
1873-
* Worker-computed aggregate value for the "Set" aggregation kind. The only possible value type is a list of Values whose type can be Long, Double, or String, according to the metric's type. All Values in the list must be of the same type.
1942+
* Worker-computed aggregate value for the "Set" aggregation kind. The only possible value type is a list of Values whose type can be Long, Double, String, or BoundedTrie according to the metric's type. All Values in the list must be of the same type.
18741943
*/
18751944
set?: any | null;
1945+
/**
1946+
* Worker-computed aggregate value for the "Trie" aggregation kind. The only possible value type is a BoundedTrieNode.
1947+
*/
1948+
trie?: any | null;
18761949
/**
18771950
* Timestamp associated with the metric value. Optional when workers are reporting work progress; it will be filled in responses from the metrics API.
18781951
*/
@@ -1890,6 +1963,10 @@ export namespace dataflow_v1b3 {
18901963
* Optional. Set of metric labels for this metric.
18911964
*/
18921965
metricLabels?: {[key: string]: string} | null;
1966+
/**
1967+
* Non-cumulative int64 value of this metric.
1968+
*/
1969+
valueGauge64?: Schema$DataflowGaugeValue;
18931970
/**
18941971
* Histogram value of this metric.
18951972
*/
@@ -2391,6 +2468,10 @@ export namespace dataflow_v1b3 {
23912468
* CPU utilization samples.
23922469
*/
23932470
cpuTime?: Schema$CPUTime[];
2471+
/**
2472+
* Optional. GPU usage samples.
2473+
*/
2474+
gpuUsage?: Schema$GPUUsage[];
23942475
/**
23952476
* Memory utilization samples.
23962477
*/
@@ -2401,7 +2482,7 @@ export namespace dataflow_v1b3 {
24012482
*/
24022483
export interface Schema$ResourceUtilizationReportResponse {}
24032484
/**
2404-
* The environment values to set at runtime. LINT.IfChange
2485+
* The environment values to set at runtime.
24052486
*/
24062487
export interface Schema$RuntimeEnvironment {
24072488
/**

0 commit comments

Comments
 (0)