Skip to content

add graphman config check, place, pools to graphql api #5751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Prev Previous commit
Next Next commit
core,node,server: add graphman config pools to graphql api
  • Loading branch information
shiyasmohd committed Jan 8, 2025
commit 91432b98c4311b970629e7a3fc6e1fa76e6cc297
1 change: 1 addition & 0 deletions core/graphman/src/commands/config/mod.rs
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
pub mod check;
pub mod place;
pub mod pools;
81 changes: 81 additions & 0 deletions core/graphman/src/commands/config/pools.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
use std::collections::BTreeMap;

use anyhow::Error as AnyError;
use async_graphql::SimpleObject;
use graph::prelude::NodeId;
use thiserror::Error;

use crate::config::Config;

#[derive(Debug, Error)]
pub enum PoolsError {
#[error("illegal node name `{0}`")]
IllegalNodeName(String),
#[error(transparent)]
Common(#[from] AnyError),
}

#[derive(Clone, Debug, SimpleObject)]
pub struct Pools {
pub shards: BTreeMap<String, u32>,
pub node: String,
}

#[derive(Debug)]
pub struct PoolsResult {
pub pools: Vec<Pools>,
pub shards: BTreeMap<String, u32>,
}

pub fn pools(config: &Config, nodes: &Vec<String>) -> Result<PoolsResult, PoolsError> {
// Quietly replace `-` with `_` in node names to make passing in pod names
// from k8s less annoying
let nodes: Vec<_> = nodes
.into_iter()
.map(|name| {
NodeId::new(name.replace('-', "_"))
.map_err(|()| PoolsError::IllegalNodeName(name.to_string()))
})
.collect::<Result<_, _>>()?;
// node -> shard_name -> size
let mut sizes = BTreeMap::new();
for node in &nodes {
let mut shard_sizes = BTreeMap::new();
for (name, shard) in &config.stores {
let size = shard
.pool_size
.size_for(node, name)
.map_err(PoolsError::Common)?;
shard_sizes.insert(name.to_string(), size);
for (replica_name, replica) in &shard.replicas {
let qname = format!("{}.{}", name, replica_name);
let size = replica
.pool_size
.size_for(node, &qname)
.map_err(PoolsError::Common)?;
shard_sizes.insert(qname, size);
}
}
sizes.insert(node.to_string(), shard_sizes);
}

let mut by_shard: BTreeMap<String, u32> = BTreeMap::new();
for shard_sizes in sizes.values() {
for (shard_name, size) in shard_sizes {
*by_shard.entry(shard_name.to_string()).or_default() += size;
}
}
let mut pools: Vec<Pools> = Vec::new();
for node in &nodes {
let empty = BTreeMap::new();
let node_sizes = sizes.get(node.as_str()).unwrap_or(&empty);
pools.push(Pools {
shards: node_sizes.clone(),
node: node.to_string(),
})
}
Ok(PoolsResult {
pools,
shards: by_shard,
})
}
4 changes: 3 additions & 1 deletion node/src/bin/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1185,7 +1185,9 @@ async fn main() -> anyhow::Result<()> {
commands::config_cmd::place::run(&ctx.config.deployment, &name, &network)
}
Check { print } => commands::config_cmd::check::run(&ctx.config, print),
Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard),
Pools { nodes, shard } => {
commands::config_cmd::pools::run(&ctx.config, nodes, shard)
}
Provider { features, network } => {
let logger = ctx.logger.clone();
let registry = ctx.registry.clone();
Expand Down
1 change: 1 addition & 0 deletions node/src/manager/commands/config_cmd/mod.rs
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
pub mod check;
pub mod place;
pub mod pools;
22 changes: 22 additions & 0 deletions node/src/manager/commands/config_cmd/pools.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
use anyhow::Error;
use graphman::{commands::config::pools::pools, config::Config};

pub fn run(config: &Config, nodes: Vec<String>, shard: bool) -> Result<(), Error> {
if nodes.is_empty() {
return Err(Error::msg("No nodes specified"));
}
let res = pools(config, &nodes)?;
if shard {
for shard in res.shards {
println!("{}: {}", shard.0, shard.1);
}
} else {
for pool in res.pools {
println!("{}:", pool.node);
for shard in pool.shards {
println!(" {}: {}", shard.0, shard.1);
}
}
}
Ok(())
}
2 changes: 2 additions & 0 deletions server/graphman/src/entities/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ mod empty_response;
mod execution;
mod execution_id;
mod place_response;
mod pools;
mod subgraph_health;

pub use self::block_hash::BlockHash;
Expand All @@ -26,4 +27,5 @@ pub use self::empty_response::EmptyResponse;
pub use self::execution::Execution;
pub use self::execution_id::ExecutionId;
pub use self::place_response::PlaceResponse;
pub use self::pools::PoolsResponse;
pub use self::subgraph_health::SubgraphHealth;
12 changes: 12 additions & 0 deletions server/graphman/src/entities/pools.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
use std::collections::BTreeMap;

use async_graphql::SimpleObject;
use graphman::commands::config::pools::Pools;

#[derive(Clone, Debug, SimpleObject)]
pub struct PoolsResponse {
/// Size of database pools for each node.
pub pools: Vec<Pools>,
/// Connections by shard rather than by node
pub shards: BTreeMap<String, u32>,
}
10 changes: 10 additions & 0 deletions server/graphman/src/resolvers/config_query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ use async_graphql::Result;

use crate::entities::ConfigCheckResponse;
use crate::entities::PlaceResponse;
use crate::entities::PoolsResponse;

mod check;
mod place;
mod pools;
pub struct ConfigQuery;

#[Object]
Expand All @@ -23,4 +25,12 @@ impl ConfigQuery {
) -> Result<PlaceResponse> {
place::run(subgraph, network).await
}

// Information about the size of database pools
pub async fn pools(
&self,
#[graphql(desc = "The names of the nodes that are going to run")] nodes: Vec<String>,
) -> Result<PoolsResponse> {
pools::run(nodes).await
}
}
19 changes: 19 additions & 0 deletions server/graphman/src/resolvers/config_query/pools.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
use async_graphql::Result;
use graphman::commands::config::pools::pools;

use crate::entities::PoolsResponse;

use super::check::fetch_config;

pub async fn run(nodes: Vec<String>) -> Result<PoolsResponse> {
let config = fetch_config()?;
let res = pools(&config, &nodes);

match res {
Ok(res) => Ok(PoolsResponse {
pools: res.pools,
shards: res.shards,
}),
Err(e) => Err(e.into()),
}
}
Loading