add graphman config check, place, pools to graphql api

core/graphman/Cargo.toml

@@ -5,10 +5,18 @@ edition.workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
+async-graphql = { workspace = true }
+clap = { workspace = true }
 diesel = { workspace = true }
+git-testament = "0.2"
 graph = { workspace = true }
 graph-store-postgres = { workspace = true }
 graphman-store = { workspace = true }
 itertools = { workspace = true }
+lazy_static = "1.5.0"
 thiserror = { workspace = true }
 tokio = { workspace = true }
+graph-chain-ethereum = { path = "../../chain/ethereum" }
+serde = { workspace = true }
+url = "2.5.2"
+shellexpand = "3.1.0"

core/graphman/src/commands/config/check.rs

@@ -0,0 +1,42 @@
+use crate::config::Config;
+use anyhow::Error;
+use graph::{components::subgraph::Settings, env::EnvVars};
+
+pub struct ConfigCheckResult {
+    pub validated: bool,
+    pub validated_subgraph_settings: bool,
+    pub config_json: Option<String>,
+}
+
+pub fn check(config: &Config, print: bool) -> Result<ConfigCheckResult, Error> {
+    let mut result = ConfigCheckResult {
+        validated: false,
+        validated_subgraph_settings: false,
+        config_json: None,
+    };
+    if print {
+        match config.to_json() {
+            Ok(txt) => {
+                result.config_json = Some(txt);
+            }
+            Err(err) => return Err(anyhow::format_err!("error serializing config: {}", err)),
+        };
+    }
+    let env_vars = EnvVars::from_env().unwrap();
+    if let Some(path) = &env_vars.subgraph_settings {
+        match Settings::from_file(path) {
+            Ok(_) => {
+                result.validated_subgraph_settings = true;
+            }
+            Err(e) => {
+                return Err(anyhow::format_err!(
+                    "configuration error in subgraph settings {}: {}",
+                    path,
+                    e
+                ));
+            }
+        }
+    };
+    result.validated = true;
+    Ok(result)
+}

core/graphman/src/commands/config/mod.rs

@@ -0,0 +1,3 @@
+pub mod check;
+pub mod place;
+pub mod pools;

core/graphman/src/commands/config/place.rs

@@ -0,0 +1,32 @@
+use anyhow::{anyhow, Error as AnyError};
+use graph_store_postgres::DeploymentPlacer;
+use thiserror::Error;
+
+pub struct PlaceResult {
+    pub shards: Vec<String>,
+    pub nodes: Vec<String>,
+}
+
+#[derive(Debug, Error)]
+pub enum PlaceError {
+    #[error("No matching placement rule; default placement from JSON RPC call would be used")]
+    NoPlacementRule,
+
+    #[error(transparent)]
+    Common(#[from] AnyError),
+}
+
+pub fn place(
+    placer: &dyn DeploymentPlacer,
+    name: &str,
+    network: &str,
+) -> Result<PlaceResult, PlaceError> {
+    match placer.place(name, network).map_err(|s| anyhow!(s))? {
+        None => Err(PlaceError::NoPlacementRule),
+        Some((shards, nodes)) => {
+            let nodes: Vec<_> = nodes.into_iter().map(|n| n.to_string()).collect();
+            let shards: Vec<_> = shards.into_iter().map(|s| s.to_string()).collect();
+            Ok(PlaceResult { shards, nodes })
+        }
+    }
+}

core/graphman/src/commands/config/pools.rs

@@ -0,0 +1,81 @@
+use std::collections::BTreeMap;
+
+use anyhow::Error as AnyError;
+use async_graphql::SimpleObject;
+use graph::prelude::NodeId;
+use thiserror::Error;
+
+use crate::config::Config;
+
+#[derive(Debug, Error)]
+pub enum PoolsError {
+    #[error("illegal node name `{0}`")]
+    IllegalNodeName(String),
+    #[error(transparent)]
+    Common(#[from] AnyError),
+}
+
+#[derive(Clone, Debug, SimpleObject)]
+pub struct Pools {
+    pub shards: BTreeMap<String, u32>,
+    pub node: String,
+}
+
+#[derive(Debug)]
+pub struct PoolsResult {
+    pub pools: Vec<Pools>,
+    pub shards: BTreeMap<String, u32>,
+}
+
+pub fn pools(config: &Config, nodes: &Vec<String>) -> Result<PoolsResult, PoolsError> {
+    // Quietly replace `-` with `_` in node names to make passing in pod names
+    // from k8s less annoying
+    let nodes: Vec<_> = nodes
+        .into_iter()
+        .map(|name| {
+            NodeId::new(name.replace('-', "_"))
+                .map_err(|()| PoolsError::IllegalNodeName(name.to_string()))
+        })
+        .collect::<Result<_, _>>()?;
+    // node -> shard_name -> size
+    let mut sizes = BTreeMap::new();
+    for node in &nodes {
+        let mut shard_sizes = BTreeMap::new();
+        for (name, shard) in &config.stores {
+            let size = shard
+                .pool_size
+                .size_for(node, name)
+                .map_err(PoolsError::Common)?;
+            shard_sizes.insert(name.to_string(), size);
+            for (replica_name, replica) in &shard.replicas {
+                let qname = format!("{}.{}", name, replica_name);
+                let size = replica
+                    .pool_size
+                    .size_for(node, &qname)
+                    .map_err(PoolsError::Common)?;
+                shard_sizes.insert(qname, size);
+            }
+        }
+        sizes.insert(node.to_string(), shard_sizes);
+    }
+
+    let mut by_shard: BTreeMap<String, u32> = BTreeMap::new();
+    for shard_sizes in sizes.values() {
+        for (shard_name, size) in shard_sizes {
+            *by_shard.entry(shard_name.to_string()).or_default() += size;
+        }
+    }
+    let mut pools: Vec<Pools> = Vec::new();
+    for node in &nodes {
+        let empty = BTreeMap::new();
+        let node_sizes = sizes.get(node.as_str()).unwrap_or(&empty);
+        pools.push(Pools {
+            shards: node_sizes.clone(),
+            node: node.to_string(),
+        })
+    }
+    Ok(PoolsResult {
+        pools,
+        shards: by_shard,
+    })
+}

core/graphman/src/commands/mod.rs

@@ -1 +1,2 @@
+pub mod config;
 pub mod deployment;

node/src/config.rs → core/graphman/src/config.rs

@@ -19,9 +19,9 @@ use graph::{
 use graph_chain_ethereum as ethereum;
 use graph_chain_ethereum::NodeCapabilities;
 use graph_store_postgres::{DeploymentPlacer, Shard as ShardName, PRIMARY_SHARD};
+use serde::Serialize;
 
 use graph::http::{HeaderMap, Uri};
-use serde::Serialize;
 use std::{
     collections::{BTreeMap, BTreeSet},
     fmt,

core/graphman/src/lib.rs

@@ -8,8 +8,10 @@
 mod error;
 
 pub mod commands;
+pub mod config;
 pub mod deployment;
 pub mod execution_tracker;
+pub mod opt;
 
 pub use self::error::GraphmanError;
 pub use self::execution_tracker::GraphmanExecutionTracker;

node/src/bin/manager.rs

@@ -19,7 +19,6 @@ use graph::{
 };
 use graph_chain_ethereum::EthereumAdapter;
 use graph_graphql::prelude::GraphQlRunner;
-use graph_node::config::{self, Config as Cfg};
 use graph_node::manager::color::Terminal;
 use graph_node::manager::commands;
 use graph_node::network_setup::Networks;
@@ -34,6 +33,7 @@ use graph_store_postgres::{
     connection_pool::ConnectionPool, BlockStore, NotificationSender, Shard, Store, SubgraphStore,
     SubscriptionManager, PRIMARY_SHARD,
 };
+use graphman::config::{self, Config as Cfg};
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use std::str::FromStr;
@@ -1182,10 +1182,12 @@ async fn main() -> anyhow::Result<()> {
                     Ok(())
                 }
                 Place { name, network } => {
-                    commands::config::place(&ctx.config.deployment, &name, &network)
+                    commands::config_cmd::place::run(&ctx.config.deployment, &name, &network)
+                }
+                Check { print } => commands::config_cmd::check::run(&ctx.config, print),
+                Pools { nodes, shard } => {
+                    commands::config_cmd::pools::run(&ctx.config, nodes, shard)
                 }
-                Check { print } => commands::config::check(&ctx.config, print),
-                Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard),
                 Provider { features, network } => {
                     let logger = ctx.logger.clone();
                     let registry = ctx.registry.clone();

node/src/chain.rs

@@ -1,4 +1,3 @@
-use crate::config::{Config, ProviderDetails};
 use crate::network_setup::{
     AdapterConfiguration, EthAdapterConfig, FirehoseAdapterConfig, Networks,
 };
@@ -31,6 +30,7 @@ use graph::tokio::time::timeout;
 use graph::url::Url;
 use graph_chain_ethereum::{self as ethereum, Transport};
 use graph_store_postgres::{BlockStore, ChainHeadUpdateListener};
+use graphman::config::{Config, ProviderDetails};
 use std::cmp::Ordering;
 use std::collections::BTreeMap;
 use std::sync::Arc;
@@ -240,7 +240,7 @@ pub async fn create_ethereum_networks_for_chain(
             "capabilities" => capabilities
         );
 
-        use crate::config::Transport::*;
+        use graphman::config::Transport::*;
 
         let transport = match web3.transport {
             Rpc => Transport::new_rpc(
@@ -553,13 +553,13 @@ pub async fn networks_as_chains(
 
 #[cfg(test)]
 mod test {
-    use crate::config::{Config, Opt};
     use crate::network_setup::{AdapterConfiguration, Networks};
     use graph::components::network_provider::ChainName;
     use graph::endpoint::EndpointMetrics;
     use graph::log::logger;
     use graph::prelude::{tokio, MetricsRegistry};
     use graph_chain_ethereum::NodeCapabilities;
+    use graphman::config::{Config, Opt};
     use std::sync::Arc;
 
     #[tokio::test]

node/src/lib.rs

@@ -6,9 +6,7 @@ use graph::{prelude::MetricsRegistry, prometheus::Registry};
 extern crate diesel;
 
 pub mod chain;
-pub mod config;
 pub mod network_setup;
-pub mod opt;
 pub mod store_builder;
 
 pub mod manager;

node/src/main.rs

@@ -20,9 +20,7 @@ use graph_core::{
     SubgraphRegistrar as IpfsSubgraphRegistrar,
 };
 use graph_graphql::prelude::GraphQlRunner;
-use graph_node::config::Config;
 use graph_node::network_setup::Networks;
-use graph_node::opt;
 use graph_node::store_builder::StoreBuilder;
 use graph_server_http::GraphQLServer as GraphQLQueryServer;
 use graph_server_index_node::IndexNodeServer;
@@ -32,6 +30,8 @@ use graph_server_websocket::SubscriptionServer as GraphQLSubscriptionServer;
 use graph_store_postgres::connection_pool::ConnectionPool;
 use graph_store_postgres::Store;
 use graph_store_postgres::{register_jobs as register_store_jobs, NotificationSender};
+use graphman::config::Config;
+use graphman::opt;
 use graphman_server::GraphmanServer;
 use graphman_server::GraphmanServerConfig;
 use std::io::{BufRead, BufReader};

node/src/manager/commands/config.rs

@@ -16,7 +16,8 @@ use graph::{
 use graph_chain_ethereum::NodeCapabilities;
 use graph_store_postgres::DeploymentPlacer;
 
-use crate::{config::Config, network_setup::Networks};
+use crate::network_setup::Networks;
+use graphman::config::Config;
 
 pub fn place(placer: &dyn DeploymentPlacer, name: &str, network: &str) -> Result<(), Error> {
     match placer.place(name, network).map_err(|s| anyhow!(s))? {

node/src/manager/commands/config_cmd/check.rs

@@ -0,0 +1,26 @@
+use anyhow::Error;
+use graphman::{commands::config::check::check, config::Config};
+
+pub fn run(config: &Config, print: bool) -> Result<(), Error> {
+    let check = check(config, print);
+
+    match check {
+        Ok(res) => {
+            if res.validated {
+                println!("Successfully validated configuration");
+            }
+            if res.validated_subgraph_settings {
+                println!("Successfully validated subgraph settings");
+            }
+            if let Some(txt) = res.config_json {
+                println!("{}", txt);
+            }
+        }
+        Err(e) => {
+            eprintln!("configuration error: {}", e);
+            std::process::exit(1);
+        }
+    }
+
+    Ok(())
+}

node/src/manager/commands/config_cmd/mod.rs

@@ -0,0 +1,3 @@
+pub mod check;
+pub mod place;
+pub mod pools;