Merge remote-tracking branch 'origin/development' into merge_web_api_…

…testing_development

# Conflicts:
#	.gitignore
#	README.md
#	src/hackingBuddyGPT/cli/wintermute.py
#	src/hackingBuddyGPT/usecases/base.py
#	src/hackingBuddyGPT/usecases/web/simple.py
#	src/hackingBuddyGPT/usecases/web/with_explanation.py
#	src/hackingBuddyGPT/usecases/web_api_testing/simple_openapi_documentation.py
#	src/hackingBuddyGPT/usecases/web_api_testing/simple_web_api_testing.py
#	src/hackingBuddyGPT/utils/db_storage/db_storage.py

.devcontainer/devcontainer.json

@@ -0,0 +1,3 @@
+{
+	"onCreateCommand": "./scripts/codespaces_create_and_start_containers.sh"
+}

.env.example

@@ -8,7 +8,12 @@ conn.port=2222
 
 # exchange with the user for your target VM
 conn.username='bob'
+#To just use keyauth only, use '' with no space for conn.password 
+#Otherwise, insert the password for instance here
 conn.password='secret'
+#To just use username and password auth only, use '' with no space for conn.keyfilename
+#Otherwise, insert the filepath for the keyfile here (for example, '/home/bob/.ssh/sshkey.rsa')
+conn.keyfilename=''
 
 # which LLM model to use (can be anything openai supports, or if you use a custom llm.api_url, anything your api provides for the model parameter
 llm.model='gpt-3.5-turbo'

.env.example.aws

@@ -0,0 +1,23 @@
+llm.api_key='your-openai-key'
+log_db.connection_string='log_db.sqlite3'
+
+# exchange with the IP of your target VM
+conn.host='enter the public IP of AWS Instance'
+conn.hostname='DNS of AWS Instance '
+conn.port=22
+
+# user of target AWS Instance
+conn.username='bob'
+#To just use keyauth only, use '' with no space for conn.password 
+#Otherwise, insert the password for instance here
+conn.password=''
+#To just use username and password auth only, use '' with no space for conn.keyfilename
+#Otherwise, insert the filepath for the keyfile here (for example, '/home/bob/.ssh/awskey.pem')
+conn.keyfilename='/home/bob/.ssh/awskey.pem'
+
+# which LLM model to use (can be anything openai supports, or if you use a custom llm.api_url, anything your api provides for the model parameter
+llm.model='gpt-3.5-turbo'
+llm.context_size=16385
+
+# how many rounds should this thing go?
+max_turns = 20

.gitignore

@@ -1,5 +1,6 @@
 .env
 venv/
+.venv/
 __pycache__/
 *.swp
 *.log
@@ -13,6 +14,18 @@ dist/
 .coverage
 src/hackingBuddyGPT/usecases/web_api_testing/openapi_spec/
 src/hackingBuddyGPT/usecases/web_api_testing/converted_files/
+/src/hackingBuddyGPT/usecases/web_api_testing/documentation/openapi_spec/
+/src/hackingBuddyGPT/usecases/web_api_testing/documentation/reports/
+scripts/codespaces_ansible.cfg
+scripts/codespaces_ansible_hosts.ini
+scripts/codespaces_ansible_id_rsa
+scripts/codespaces_ansible_id_rsa.pub
+scripts/mac_ansible.cfg
+scripts/mac_ansible_hosts.ini
+scripts/mac_ansible_id_rsa
+scripts/mac_ansible_id_rsa.pub
+.aider*
+
 src/hackingBuddyGPT/usecases/web_api_testing/documentation/openapi_spec/
 src/hackingBuddyGPT/usecases/web_api_testing/documentation/reports/
 src/hackingBuddyGPT/usecases/web_api_testing/retrieve_spotify_token.py

CODESPACES.md

@@ -0,0 +1,179 @@
+# Use Case: GitHub Codespaces
+
+**Backstory**
+
+https://github.com/ipa-lab/hackingBuddyGPT/pull/85#issuecomment-2331166997
+
+> Would it be possible to add codespace support to hackingbuddygpt in a way, that only spawns a single container (maybe with the suid/sudo use-case) and starts hackingBuddyGPT against that container? That might be the 'easiest' show-case/use-case for a new user.
+
+**Steps**
+1. Go to https://github.com/ipa-lab/hackingBuddyGPT
+2. Click the "Code" button.
+3. Click the "Codespaces" tab.
+4. Click the "Create codespace on main" button.
+5. Wait for Codespaces to start — This may take upwards of 10 minutes.
+
+> Setting up remote connection: Building codespace...
+
+6. After Codespaces started, you may need to restart a new Terminal via the Command Palette:
+
+Press the key combination:
+
+> `⇧⌘P` `Shift+Command+P` (Mac) / `Ctrl+Shift+P` (Windows/Linux)
+
+In the Command Palette, type `>` and `Terminal: Create New Terminal` and press the return key.
+
+7. You should see a new terminal similar to the following:
+
+> 👋 Welcome to Codespaces! You are on our default image.
+>
+>    `-` It includes runtimes and tools for Python, Node.js, Docker, and more. See the full list here: https://aka.ms/ghcs-default-image
+>
+>    `-` Want to use a custom image instead? Learn more here: https://aka.ms/configure-codespace
+>
+> 🔍 To explore VS Code to its fullest, search using the Command Palette (Cmd/Ctrl + Shift + P or F1).
+>
+> 📝 Edit away, run your app as usual, and we'll automatically make it available for you to access.
+>
+> @github-username ➜ /workspaces/hackingBuddyGPT (main) $
+
+Type the following to manually run:
+```bash
+./scripts/codespaces_start_hackingbuddygpt_against_a_container.sh
+```
+7. Eventually, you should see:
+
+> Currently, May 2024, running hackingBuddyGPT with GPT-4-turbo against a benchmark containing 13 VMs (with maximum 20 tries per VM) cost around $5.
+>
+> Therefore, running hackingBuddyGPT with GPT-4-turbo against containing a container with maximum 10 tries would cost around $0.20.
+>
+> Enter your OpenAI API key and press the return key:
+
+8. As requested, please enter your OpenAI API key and press the return key.
+
+9. hackingBuddyGPT should start:
+
+> Starting hackingBuddyGPT against a container...
+
+10. If your OpenAI API key is *valid*, then you should see output similar to the following:
+
+> [00:00:00] Starting turn 1 of 10
+>
+> Got command from LLM:
+>
+> …
+>
+> [00:01:00] Starting turn 10 of 10
+>
+> …
+>
+> Run finished
+>
+> maximum turn number reached
+
+11. If your OpenAI API key is *invalid*, then you should see output similar to the following:
+
+> [00:00:00] Starting turn 1 of 10
+>
+> Traceback (most recent call last):
+>
+> …
+>
+> Exception: Error from OpenAI Gateway (401
+
+12. Alternatively, use Google Gemini instead of OpenAI
+
+**Preqrequisites:**
+
+```bash
+python -m venv venv
+```
+
+```bash
+source ./venv/bin/activate
+```
+
+```bash
+pip install -e .
+```
+
+**Use gemini-openai-proxy and Gemini:**
+
+http://localhost:8080 is gemini-openai-proxy
+
+`gpt-4` maps to `gemini-1.5-flash-latest`
+
+Hence use `gpt-4` below in `--llm.model=gpt-4`
+
+Gemini free tier has a limit of 15 requests per minute, and 1500 requests per day
+
+Hence `--max_turns 999999999` will exceed the daily limit
+
+**Run gemini-openai-proxy**
+
+```bash
+docker run --restart=unless-stopped -it -d -p 8080:8080 --name gemini zhu327/gemini-openai-proxy:latest
+```
+
+**Manually enter your GEMINI_API_KEY value based on** https://aistudio.google.com/app/apikey
+
+```bash
+export GEMINI_API_KEY=
+```
+
+**Starting hackingBuddyGPT against a container...**
+
+```bash
+wintermute LinuxPrivesc --llm.api_key=$GEMINI_API_KEY --llm.model=gpt-4 --llm.context_size=1000000 --conn.host=192.168.122.151 --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1 --llm.api_url=http://localhost:8080 --llm.api_backoff=60 --max_turns 999999999
+```
+
+**Google AI Studio: Gemini free tier has a limit of 15 requests per minute, and 1500 requests per day:**
+
+https://ai.google.dev/pricing#1_5flash
+
+> Gemini 1.5 Flash
+>
+> The Gemini API “free tier” is offered through the API service with lower rate limits for testing purposes. Google AI Studio usage is completely free in all available countries.
+>
+> Rate Limits
+>
+> 15 RPM (requests per minute)
+>
+> 1 million TPM (tokens per minute)
+>
+> 1,500 RPD (requests per day)
+>
+> Used to improve Google's products
+>
+> Yes
+
+https://ai.google.dev/gemini-api/terms#data-use-unpaid
+
+> How Google Uses Your Data
+>
+> When you use Unpaid Services, including, for example, Google AI Studio and the unpaid quota on Gemini API, Google uses the content you submit to the Services and any generated responses to provide, improve, and develop Google products and services and machine learning technologies, including Google's enterprise features, products, and services, consistent with our Privacy Policy https://policies.google.com/privacy
+>
+> To help with quality and improve our products, human reviewers may read, annotate, and process your API input and output. Google takes steps to protect your privacy as part of this process. This includes disconnecting this data from your Google Account, API key, and Cloud project before reviewers see or annotate it. **Do not submit sensitive, confidential, or personal information to the Unpaid Services.**
+
+**README.md and Disclaimers:**
+
+https://github.com/ipa-lab/hackingBuddyGPT/blob/main/README.md
+
+**Please refer to [README.md](https://github.com/ipa-lab/hackingBuddyGPT/blob/main/README.md) for all disclaimers.**
+
+Please note and accept all of them.
+
+**References:**
+* https://docs.github.com/en/codespaces
+* https://docs.github.com/en/codespaces/getting-started/quickstart
+* https://docs.github.com/en/codespaces/reference/using-the-vs-code-command-palette-in-codespaces
+* https://openai.com/api/pricing/
+* https://platform.openai.com/docs/quickstart
+* https://platform.openai.com/api-keys
+* https://ai.google.dev/gemini-api/docs/ai-studio-quickstart
+* https://aistudio.google.com/
+* https://aistudio.google.com/app/apikey
+* https://ai.google.dev/
+* https://ai.google.dev/gemini-api/docs/api-key
+* https://github.com/zhu327/gemini-openai-proxy
+* https://hub.docker.com/r/zhu327/gemini-openai-proxy

MAC.md

@@ -0,0 +1,129 @@
+## Use Case: Mac, Docker Desktop and Gemini-OpenAI-Proxy
+
+**Docker Desktop runs containers in a virtual machine on Mac.**
+
+**Run hackingBuddyGPT on Mac as follows:**
+
+Target a localhost container ansible-ready-ubuntu
+
+via Docker Desktop https://docs.docker.com/desktop/setup/install/mac-install/
+
+and Gemini-OpenAI-Proxy https://github.com/zhu327/gemini-openai-proxy
+
+There are bugs in Docker Desktop on Mac that prevent creation of a custom Docker network 192.168.65.0/24
+
+Therefore, localhost TCP port 49152 (or higher) dynamic port number is used for an ansible-ready-ubuntu container
+
+http://localhost:8080 is gemini-openai-proxy
+
+gpt-4 maps to gemini-1.5-flash-latest
+
+Hence use gpt-4 below in --llm.model=gpt-4
+
+Gemini free tier has a limit of 15 requests per minute, and 1500 requests per day
+
+Hence --max_turns 999999999 will exceed the daily limit
+
+For example:
+
+```zsh
+export GEMINI_API_KEY=
+
+export PORT=49152
+
+wintermute LinuxPrivesc --llm.api_key=$GEMINI_API_KEY --llm.model=gpt-4 --llm.context_size=1000000 --conn.host=localhost --conn.port $PORT --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1 --llm.api_url=http://localhost:8080 --llm.api_backoff=60 --max_turns 999999999
+```
+
+The above example is consolidated into shell scripts with prerequisites as follows:
+
+**Preqrequisite: Install Homebrew and Bash version 5:**
+
+```zsh
+/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+```
+
+**Install Bash version 5 via Homebrew:**
+
+```zsh
+brew install bash
+```
+
+Bash version 4 or higher is needed for `scripts/mac_create_and_start_containers.sh`
+
+Homebrew provides GNU Bash version 5 via license GPLv3+
+
+Whereas Mac provides Bash version 3 via license GPLv2
+
+**Create and start containers:**
+
+```zsh
+./scripts/mac_create_and_start_containers.sh
+```
+
+**Start hackingBuddyGPT against a container:**
+
+```zsh
+export GEMINI_API_KEY=
+```
+
+```zsh
+./scripts/mac_start_hackingbuddygpt_against_a_container.sh
+```
+
+**Troubleshooting:**
+
+**Docker Desktop: Internal Server Error**
+
+```zsh
+Server:
+ERROR: request returned Internal Server Error for API route and version http://%2FUsers%2Fusername%2F.docker%2Frun%2Fdocker.sock/v1.47/info, check if the server supports the requested API version
+errors pretty printing info
+```
+
+You may need to uninstall Docker Desktop https://docs.docker.com/desktop/uninstall/ and reinstall it from https://docs.docker.com/desktop/setup/install/mac-install/ and try again.
+
+Alternatively, restart Docker Desktop and try again.
+
+**There are known issues with Docker Desktop on Mac, such as:**
+
+* Bug: Docker CLI Hangs for all commands
+https://github.com/docker/for-mac/issues/6940
+
+* Regression: Docker does not recover from resource saver mode
+https://github.com/docker/for-mac/issues/6933
+
+**Google AI Studio: Gemini free tier has a limit of 15 requests per minute, and 1500 requests per day:**
+
+https://ai.google.dev/pricing#1_5flash
+
+> Gemini 1.5 Flash
+>
+> The Gemini API “free tier” is offered through the API service with lower rate limits for testing purposes. Google AI Studio usage is completely free in all available countries.
+>
+> Rate Limits
+>
+> 15 RPM (requests per minute)
+>
+> 1 million TPM (tokens per minute)
+>
+> 1,500 RPD (requests per day)
+>
+> Used to improve Google's products
+>
+> Yes
+
+https://ai.google.dev/gemini-api/terms#data-use-unpaid
+
+> How Google Uses Your Data
+>
+> When you use Unpaid Services, including, for example, Google AI Studio and the unpaid quota on Gemini API, Google uses the content you submit to the Services and any generated responses to provide, improve, and develop Google products and services and machine learning technologies, including Google's enterprise features, products, and services, consistent with our Privacy Policy https://policies.google.com/privacy
+>
+> To help with quality and improve our products, human reviewers may read, annotate, and process your API input and output. Google takes steps to protect your privacy as part of this process. This includes disconnecting this data from your Google Account, API key, and Cloud project before reviewers see or annotate it. **Do not submit sensitive, confidential, or personal information to the Unpaid Services.**
+
+**README.md and Disclaimers:**
+
+https://github.com/ipa-lab/hackingBuddyGPT/blob/main/README.md
+
+**Please refer to [README.md](https://github.com/ipa-lab/hackingBuddyGPT/blob/main/README.md) for all disclaimers.**
+
+Please note and accept all of them.