Merge branch 'master' into update-gopacket-vendor

Author: RickdeJager

.env.example

@@ -2,44 +2,96 @@
 # Tulip config
 ##############################
 
-# The connection string to connect to the mongo bd
-TULIP_MONGO="mongo:27017"
+# Timescale connection
+TIMESCALE="postgres://tulip@timescale:5432/tulip"
+
 # The location of your pcaps as seen by the host
 TRAFFIC_DIR_HOST="./services/test_pcap"
+
 # The location of your pcaps (and eve.json), as seen by the container
 TRAFFIC_DIR_DOCKER="/traffic"
 
+# Visualizer
+VISUALIZER_URL="http://scraper.example.com"
+
 ##############################
 # Game config
 ##############################
 
 # Start time of the CTF (or network open if you prefer)
-TICK_START="2018-06-27T13:00+02:00"
+TICK_START="2024-11-30T13:00:00Z"
+
 # Tick length in ms
 TICK_LENGTH=180000
+
 # The flag format in regex
 FLAG_REGEX="[A-Z0-9]{31}="
 
+# VM IP (inside gamenet)
+# Currently ignored unless FLAGID_SCRAPE is set
+VM_IP="10.10.3.1"
+TEAM_ID="3"
+
 ##############################
 # PCAP_OVER_IP CONFIGS
 ##############################
 
+# Enable pcap-over-ip and choose server endpoint
+# Empty value = disabled
+PCAP_OVER_IP=
 #PCAP_OVER_IP="host.docker.internal:1337"
-# # For multiple PCAP_OVER_IP you can comma separate
+# For multiple PCAP_OVER_IP you can comma separate
 #PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050"
 
+##############################
+# DUMP_PCAPS CONFIGS
+##############################
+
+# Enable pcap dumping and select target location
+# Empty value = disabled
+DUMP_PCAPS=
+#DUMP_PCAPS="/traffic"
+
+# Dumping options
+# Ignored unless DUMP_PCAPS is set
+DUMP_PCAPS_INTERVAL="1m"
+DUMP_PCAPS_FILENAME="2006-01-02_15-04-05.pcap"
+
 ##############################
 # FLAGID CONFIGS
 ##############################
 
-# # enable flagid scrapping
-# FLAGID_SCRAPE=1
-# # enable flagid scanning
-# FLAGID_SCAN=1
-# # Flag Lifetime in Ticks (-1 for no check, pls don't use outside testing)
-# FLAG_LIFETIME=-1
-# # Flagid endpoint currently Testendpoint in docker compose
-# FLAGID_ENDPOINT="http://flagidendpoint:8000/flagids.json"
-# # VM IP (inside gamenet)
-# VM_IP="10.10.3.1"
-# TEAM_ID="10.10.3.1"
+# Enable flagid scrapping
+# Empty value = disabled
+FLAGID_SCRAPE=
+#FLAGID_SCRAPE=1
+
+# Enable flagid scanning - Tags flag ids in traffic
+# Empty value = disabled
+# Does nothing unless FLAGID_SCRAPE is set
+FLAGID_SCAN=
+#FLAGID_SCAN=1
+
+# Flag lifetime in ticks
+# Empty value = Fallback to TICK_LENGTH
+# -1 = No check, pls don't use outside testing
+FLAG_LIFETIME=
+#FLAG_LIFETIME=-1
+#FLAG_LIFETIME=5
+
+# Flagid endpoint
+# Default value is a test container in docker-compose-test.yml, change this for production
+# Ignored unless FLAGID_SCRAPE is set
+FLAGID_ENDPOINT="http://flagidendpoint:8000/flagids.json"
+
+##############################
+# FLAG_VALIDATOR CONFIGS
+##############################
+
+# Enables flag validation / fake flag feature. Must be one of: faust, enowars, eno, itad
+# Empty value = disabled
+FLAG_VALIDATOR_TYPE=
+
+# Some flag validators can make use of (our) team number/ID
+# Ignored unless FLAG_VALIDATOR_TYPE is set
+FLAG_VALIDATOR_TEAM=42

.gitignore

@@ -21,7 +21,7 @@ Tulip was developed by Team Europe for use in the first International Cyber Secu
 ![](./demo_images/demo3.png)
 
 ## Configuration
-Before starting the stack, edit `services/configurations.py`:
+Before starting the stack, edit `services/api/configurations.py`:
 
 ```
 vm_ip = "10.60.4.1"

README.md

@@ -1,12 +1,23 @@
-version: "3.2"
+version: "3.5"
 services:
-  mongo:
-    image: mongo:5
+  timescale:
+    build: services/timescale
+    image: tulip-timescale:latest
+    restart: unless-stopped
+    volumes:
+    - timescale-data:/var/lib/postgresql/data
+    - ./services/schema/system.sql:/docker-entrypoint-initdb.d/100_system.sql:ro
+    - ./services/schema/functions.sql:/docker-entrypoint-initdb.d/101_functions.sql:ro
+    - ./services/schema/schema.sql:/docker-entrypoint-initdb.d/102_schema.sql:ro
     networks:
       - internal
-    restart: always
-    ports:
-      - "27017:27017"
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_USER: tulip
+      POSTGRES_DB: tulip
+    # This does not need to be adjusted, unless you actually want to limit it
+    # Postgres uses shared memory for caching, and docker assigns just 64 MB by default
+    shm_size: '128g'
 
   frontend:
     build:
@@ -16,13 +27,16 @@ services:
     restart: unless-stopped
     ports:
       - "3000:3000"
+    expose:
+      - 3000
     depends_on:
-      - mongo
+      - timescale
       - api
     networks:
       - internal
     environment:
       API_SERVER_ENDPOINT: http://api:5000/
+      VIRTUAL_HOST: tulip.h4xx.eu
 
   api:
     build:
@@ -31,35 +45,37 @@ services:
     image: tulip-api:latest
     restart: unless-stopped
     depends_on:
-      - mongo
+      - timescale
     networks:
       - internal
     volumes:
       - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
     environment:
-      TULIP_MONGO: ${TULIP_MONGO}
+      TIMESCALE: ${TIMESCALE}
       TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER}
       FLAG_REGEX: ${FLAG_REGEX}
       TICK_START: ${TICK_START}
       TICK_LENGTH: ${TICK_LENGTH}
       VM_IP: ${VM_IP}
 
   flagids:
-    restart: on-failure
+    restart: unless-stopped
     build:
       context: services/flagids
     image: tulip-flagids:latest
     depends_on:
-      - mongo
+      - timescale
     networks:
       - internal
     environment:
-      TULIP_MONGO: ${TULIP_MONGO}
+      TIMESCALE: ${TIMESCALE}
       TICK_START: ${TICK_START}
       TICK_LENGTH: ${TICK_LENGTH}
       FLAGID_SCRAPE: ${FLAGID_SCRAPE}
       TEAM_ID: ${TEAM_ID}
       FLAGID_ENDPOINT: ${FLAGID_ENDPOINT}
+      VISUALIZER_URL: ${VISUALIZER_URL}
+      DUMP_PCAPS: ${DUMP_PCAPS}
 
   assembler:
     build:
@@ -68,38 +84,53 @@ services:
     image: tulip-assembler:latest
     restart: unless-stopped
     depends_on:
-      - mongo
+      - timescale
     networks:
       - internal
     volumes:
-      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
-    command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}"
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z
+    # Command line flags most likely to fix a tulip issue:
+    # - -http-session-tracking: enable HTTP session tracking
+    # - -dir: directory to read traffic from
+    # - -skipchecksum: skip checksum validation
+    # - -flush-after: i.e. 2m Not needed in pcap rotation mode
+    # - -disable-converters: disable converters
+    # - -discard-extra-data: dont split large flow items, just discard them
+    command: "./assembler -http-session-tracking -skipchecksum -disable-converters -dir ${TRAFFIC_DIR_DOCKER}"
     environment:
-      TULIP_MONGO: ${TULIP_MONGO}
+      TIMESCALE: ${TIMESCALE}
       FLAG_REGEX: ${FLAG_REGEX}
+      TICK_START: ${TICK_START}
       TICK_LENGTH: ${TICK_LENGTH}
       FLAGID_SCAN: ${FLAGID_SCAN}
       FLAG_LIFETIME: ${FLAG_LIFETIME}
+      FLAG_VALIDATOR_TYPE: ${FLAG_VALIDATOR_TYPE}
+      FLAG_VALIDATOR_TEAM: ${FLAG_VALIDATOR_TEAM}
       PCAP_OVER_IP: ${PCAP_OVER_IP}
+      DUMP_PCAPS: ${DUMP_PCAPS}
+      DUMP_PCAPS_INTERVAL: ${DUMP_PCAPS_INTERVAL}
+      DUMP_PCAPS_FILENAME: ${DUMP_PCAPS_FILENAME}
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-
   enricher:
     build:
       context: services/go-importer
       dockerfile: Dockerfile-enricher
     image: tulip-enricher:latest
     restart: unless-stopped
     depends_on:
-      - mongo
+      - timescale
     networks:
       - internal
     volumes:
-      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z
     command: "./enricher -eve ${TRAFFIC_DIR_DOCKER}/eve.json"
     environment:
-      TULIP_MONGO: ${TULIP_MONGO}
+      TIMESCALE: ${TIMESCALE}
+
+volumes:
+  timescale-data:
 
 networks:
   internal:

docker-compose.yml

@@ -13,4 +13,4 @@ RUN yarn run build
 
 EXPOSE 3000
 
-CMD yarn run preview --host --port 3000
+CMD ["yarn", "run", "preview", "--host", "--port", "3000"]