Merge pull request #40 from 0x707a15ec/teameurope/frontend-features

Added Suricata Docker deployment, enabled flow marking, some usability and documentation improvements

Author: RickdeJager

.env.example

@@ -11,8 +11,11 @@ TRAFFIC_DIR_HOST="./services/test_pcap"
 # The location of your pcaps (and eve.json), as seen by the container
 TRAFFIC_DIR_DOCKER="/traffic"
 
+# Set BPF filter expression (see https://www.tcpdump.org/manpages/pcap-filter.7.html)
+#BPF="port 8080"
+
 # Visualizer
-VISUALIZER_URL="http://scraper.example.com"
+#VISUALIZER_URL="http://scraper.example.com"
 
 ##############################
 # Game config
@@ -42,6 +45,7 @@ PCAP_OVER_IP=
 #PCAP_OVER_IP="host.docker.internal:1337"
 # For multiple PCAP_OVER_IP you can comma separate
 #PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050"
+<<<<<<< HEAD
 
 ##############################
 # DUMP_PCAPS CONFIGS
@@ -95,3 +99,11 @@ FLAG_VALIDATOR_TYPE=
 # Some flag validators can make use of (our) team number/ID
 # Ignored unless FLAG_VALIDATOR_TYPE is set
 FLAG_VALIDATOR_TEAM=42
+
+##############################
+# SURICATA CONFIGS
+##############################
+
+# Directory for Suricata files (see suricata/etc, suricata/lib/rules, suricata/logs)
+# (they should be generated on first run)
+#SURICATA_DIR_HOST="./suricata"

.gitignore

@@ -132,3 +132,4 @@ workspace.xml
 .idea
 
 /traffic
+suricata

README.md

@@ -13,7 +13,12 @@ Tulip was developed by Team Europe for use in the first International Cyber Secu
 * Synchronized with Suricata.
 * Flow diffing
 * Time and size-based plots for correlation.
-* Linking HTTP sessions together based on cookies (Experimental, disabled by default)
+* Linking HTTP sessions together based on cookies (Experimental*, disabled by default)
+* PCAP-over-IP with BPF filtering support**
+
+\* - to enable, add `-experimental` after `./assembler` in `docker-compose.yml`
+
+\*\* - to enable, configure PCAP-over-IP server (e.g. [pcap-broker](https://github.com/fox-it/pcap-broker) as suggested in [PR 24](https://github.com/OpenAttackDefenseTools/tulip/pull/24)) and set `PCAP_OVER_IP` (and `BPF` if necessary) in `.env`
 
 ## Screenshots
 ![](./demo_images/demo1.png)
@@ -45,21 +50,54 @@ docker-compose up -d --build
 ```
 To ingest traffic, it is recommended to create a shared bind mount with the docker-compose. One convenient way to set this up is as follows:
 1. On the vulnbox, start a rotating packet sniffer (e.g. tcpdump, suricata, ...)
-1. Using rsync, copy complete captures to the machine running tulip (e.g. to /traffic)
-1. Add a bind to the assembler service so it can read /traffic
+```bash
+tcpdump -i eth0 -G 180 -w "traffic_%H:%M:%S.pcap" port 8080
+```
+2. Using rsync, copy complete captures to the machine running tulip (e.g. to /traffic)
+```bash
+rsync -avz -e ssh --progress root@10.0.0.2:/pcaps ./pcaps
+```
+3. Add a bind to the assembler service so it can read /traffic
+   > (Just change `TRAFFIC_DIR_HOST` in `.env`)
 
 The ingestor will use inotify to watch for new pcap's and suricata logs. No need to set a chron job.
 
 
 ## Suricata synchronization
 
+### Run in Docker
+
+Configure `SURICATA_DIR_HOST` in `.env`.
+
+Create some rules (404 for testing):
+```bash
+. .env
+mkdir -p ${SURICATA_DIR_HOST}/{etc,lib/rules,log}
+echo 'alert tcp any any -> any any (msg: "404 Not Found"; http.stat_code; content:"404"; metadata: tag notfound; sid:4; rev: 1;)' >> ${SURICATA_DIR_HOST}/lib/rules/suricata.rules
+```
+
+After that run (default config for `eve.json` logging was good enough):
+
+```bash
+docker compose -f docker-compose-suricata.yml up -d --build
+```
+
 ### Metadata
 Tags are read from the metadata field of a rule. For example, here's a simple rule to detect a path traversal:
 ```
 alert tcp any any -> any any (msg: "Path Traversal-../"; flow:to_server; content: "../"; metadata: tag path_traversal; sid:1; rev: 1;)
 ```
 Once this rule is seen in traffic, the `path_traversal` tag will automatically be added to the filters in Tulip.
 
+> [!NOTE]
+>
+> After editing Suricata rules (renaming or id change) please:
+>
+> Remove old logs: `rm ${SURICATA_DIR_HOST}/log/*` (otherwise old signatures will be repopulated).
+>
+> Restart Docker containers.
+>
+> If database was only restarted (not dropped), try cleaning tags/signatures manually.
 
 ### eve.json
 Suricata alerts are read directly from the `eve.json` file. Because this file can get quite verbose when all extensions are enabled, it is recommended to strip the config down a fair bit. For example:
@@ -88,7 +126,6 @@ Your Tulip instance will probably contain sensitive CTF information, like flags
 
 # Contributing
 If you have an idea for a new feature, bug fixes, UX improvements, or other contributions, feel free to open a pull request or create an issue!      
-When opening a pull request, please target the `devel` branch.
 
 # Credits
 Tulip was written by [@RickdeJager](https://github.com/rickdejager) and [@Bazumo](https://github.com/bazumo), with additional help from [@Sijisu](https://github.com/sijisu). Thanks to our fellow Team Europe players and coaches for testing, feedback and suggestions. Finally, thanks to the team behind [flower](https://github.com/secgroup/flower) for opensourcing their tooling.

docker-compose-suricata.yml

@@ -0,0 +1,149 @@
+version: "3.5"
+services:
+  timescale:
+    build: services/timescale
+    image: tulip-timescale:latest
+    restart: unless-stopped
+    volumes:
+    - timescale-data:/var/lib/postgresql/data
+    - ./services/schema/system.sql:/docker-entrypoint-initdb.d/100_system.sql:ro
+    - ./services/schema/functions.sql:/docker-entrypoint-initdb.d/101_functions.sql:ro
+    - ./services/schema/schema.sql:/docker-entrypoint-initdb.d/102_schema.sql:ro
+    networks:
+      - internal
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_USER: tulip
+      POSTGRES_DB: tulip
+    # This does not need to be adjusted, unless you actually want to limit it
+    # Postgres uses shared memory for caching, and docker assigns just 64 MB by default
+    shm_size: '128g'
+
+  frontend:
+    build:
+      context: frontend
+      dockerfile: Dockerfile-frontend
+    image: tulip-frontend:latest
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    expose:
+      - 3000
+    depends_on:
+      - timescale
+      - api
+    networks:
+      - internal
+    environment:
+      API_SERVER_ENDPOINT: http://api:5000/
+      VIRTUAL_HOST: tulip.h4xx.eu
+
+  api:
+    build:
+      context: services/api
+      dockerfile: Dockerfile-api
+    image: tulip-api:latest
+    restart: unless-stopped
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
+    environment:
+      TIMESCALE: ${TIMESCALE}
+      TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER}
+      FLAG_REGEX: ${FLAG_REGEX}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      VM_IP: ${VM_IP}
+
+  flagids:
+    restart: unless-stopped
+    build:
+      context: services/flagids
+    image: tulip-flagids:latest
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    environment:
+      TIMESCALE: ${TIMESCALE}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      FLAGID_SCRAPE: ${FLAGID_SCRAPE}
+      TEAM_ID: ${TEAM_ID}
+      FLAGID_ENDPOINT: ${FLAGID_ENDPOINT}
+      VISUALIZER_URL: ${VISUALIZER_URL}
+      DUMP_PCAPS: ${DUMP_PCAPS}
+
+  assembler:
+    build:
+      context: services/go-importer
+      dockerfile: Dockerfile-assembler
+    image: tulip-assembler:latest
+    restart: unless-stopped
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z
+    # Command line flags most likely to fix a tulip issue:
+    # - -http-session-tracking: enable HTTP session tracking
+    # - -dir: directory to read traffic from
+    # - -skipchecksum: skip checksum validation
+    # - -flush-after: i.e. 2m Not needed in pcap rotation mode
+    # - -disable-converters: disable converters
+    # - -discard-extra-data: dont split large flow items, just discard them
+    command: "./assembler -http-session-tracking -skipchecksum -disable-converters -dir ${TRAFFIC_DIR_DOCKER}"
+    environment:
+      TIMESCALE: ${TIMESCALE}
+      FLAG_REGEX: ${FLAG_REGEX}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      FLAGID_SCAN: ${FLAGID_SCAN}
+      FLAG_LIFETIME: ${FLAG_LIFETIME}
+      FLAG_VALIDATOR_TYPE: ${FLAG_VALIDATOR_TYPE}
+      FLAG_VALIDATOR_TEAM: ${FLAG_VALIDATOR_TEAM}
+      PCAP_OVER_IP: ${PCAP_OVER_IP}
+      DUMP_PCAPS: ${DUMP_PCAPS}
+      DUMP_PCAPS_INTERVAL: ${DUMP_PCAPS_INTERVAL}
+      DUMP_PCAPS_FILENAME: ${DUMP_PCAPS_FILENAME}
+      BPF: ${BPF}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  enricher:
+    build:
+      context: services/go-importer
+      dockerfile: Dockerfile-enricher
+    image: tulip-enricher:latest
+    restart: unless-stopped
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    volumes:
+      - ${SURICATA_DIR_HOST}/log:/suricata
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
+    command: "./enricher -eve /suricata/eve.json"
+    environment:
+      TIMESCALE: ${TIMESCALE}
+
+  suricata:
+    image: jasonish/suricata:7.0
+    restart: unless-stopped
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
+      - ${SURICATA_DIR_HOST}/log:/var/log/suricata
+      - ${SURICATA_DIR_HOST}/etc:/etc/suricata
+      - ${SURICATA_DIR_HOST}/lib:/var/lib/suricata
+    environment:
+      SURICATA_OPTIONS: "-l /var/log/suricata -v -r ${TRAFFIC_DIR_DOCKER} --pcap-file-continuous"
+
+volumes:
+  timescale-data:
+
+networks:
+  internal:

docker-compose.yml

@@ -110,6 +110,7 @@ services:
       DUMP_PCAPS: ${DUMP_PCAPS}
       DUMP_PCAPS_INTERVAL: ${DUMP_PCAPS_INTERVAL}
       DUMP_PCAPS_FILENAME: ${DUMP_PCAPS_FILENAME}
+      BPF: ${BPF}
     extra_hosts:
       - "host.docker.internal:host-gateway"
 

frontend/src/api.ts

@@ -166,28 +166,28 @@ export const tulipApi = createApi({
       }),
       // TODO: optimistic cache update
 
-      // async onQueryStarted({ id, star }, { dispatch, queryFulfilled }) {
-      //   // `updateQueryData` requires the endpoint name and cache key arguments,
-      //   // so it knows which piece of cache state to update
-      //   const patchResult = dispatch(
-      //     tulipApi.util.updateQueryData("getFlows", undefined, (flows) => {
-      //       // The `flows` is Immer-wrapped and can be "mutated" like in createSlice
-      //       const flow = flows.find((flow) => flow._id.$oid === id);
-      //       if (flow) {
-      //         if (star) {
-      //           flow.tags.push("starred");
-      //         } else {
-      //           flow.tags = flow.tags.filter((tag) => tag != "starred");
-      //         }
-      //       }
-      //     })
-      //   );
-      //   try {
-      //     await queryFulfilled;
-      //   } catch {
-      //     patchResult.undo();
-      //   }
-      // },
+      async onQueryStarted({ id, star }, { dispatch, queryFulfilled }) {
+        // `updateQueryData` requires the endpoint name and cache key arguments,
+        // so it knows which piece of cache state to update
+        const patchResult = dispatch(
+          tulipApi.util.updateQueryData("getFlows", {service: "undefined", tags_include: [], tags_exclude:[]}, (flows) => {
+            // The `flows` is Immer-wrapped and can be "mutated" like in createSlice
+            const flow = flows.find((flow) => flow.id === id);
+            if (flow) {
+              if (star) {
+                flow.tags.push("starred");
+              } else {
+                flow.tags = flow.tags.filter((tag) => tag != "starred");
+              }
+            }
+          })
+        );
+        try {
+          await queryFulfilled;
+        } catch {
+          patchResult.undo();
+        }
+      },
     }),
   }),
 });

frontend/src/components/Corrie.tsx

@@ -190,6 +190,7 @@ export const Corrie = () => {
           >
             under attack
           </button>
+          <p className="text-left px-2 py-2">After clicking on a flow, press 'w' to scroll to it in flow list</p>
         </div>
       </div>
       <div className="flex-1 w-full overflow-hidden p-4">

frontend/src/components/FlowList.tsx

@@ -14,6 +14,7 @@ import {
   START_FILTER_KEY,
   END_FILTER_KEY,
   FLOW_LIST_REFETCH_INTERVAL_MS,
+  FORCE_REFETCH_ON_STAR,
 } from "../const";
 import { useAppSelector, useAppDispatch } from "../store";
 import { toggleFilterTag, toggleTagIntersectMode } from "../store/filter";
@@ -118,6 +119,7 @@ export function FlowList() {
 
   const onHeartHandler = async (flow: Flow) => {
     await starFlow({ id: flow.id, star: !flow.tags.includes("starred") });
+    if(FORCE_REFETCH_ON_STAR) refetch();
   };
 
   const navigate = useNavigate();
@@ -164,7 +166,30 @@ export function FlowList() {
     [transformedFlowData]
   )
 
+  useHotkeys('x', async () => {
+    if(transformedFlowData) {
+      let flow = transformedFlowData[flowIndex ?? 0]
+      await onHeartHandler(flow);
+    }
+  })
+
   useHotkeys('j', () => setFlowIndex(fi => Math.min((transformedFlowData?.length ?? 1)-1, fi + 1)), [transformedFlowData?.length]);
+  useHotkeys('w', () => {
+    if(transformedFlowData) {
+      let idAtIndex = transformedFlowData[flowIndex ?? 0].id;
+      if (idAtIndex != openedFlowID) {
+        let flowids = flowData?.map((flow, idx) => ([flow.id, idx]))
+        if (flowids) {
+          let found = flowids.filter((el)=>(el[0] == openedFlowID))
+          if (found.length > 0) {
+            let n = Number(found[0][1])
+            setFlowIndex(n)
+          }
+        }
+      }
+    }
+  }
+  );
   useHotkeys('k', () => setFlowIndex(fi => Math.max(0, fi - 1)));
   useHotkeys('i', () => {
     setShowFilters(true)
@@ -178,6 +203,12 @@ export function FlowList() {
       dispatch(toggleFilterTag("flag-out"))
     }
   }, [availableTags]);
+  useHotkeys('t', () => {
+    setShowFilters(true)
+    if ((availableTags ?? []).includes("starred")) {
+      dispatch(toggleFilterTag("starred"))
+    }
+  }, [availableTags]);
   useHotkeys('r', () => refetch());
 
   const [showFilters, setShowFilters] = useState(false);