Git Secrets Management: Preventing Credential Leaks in Repositories

┌─────────────────────────────────────────────────────────────┐
│                  SECRET EXPOSURE TIMELINE                    │
├─────────────────────────────────────────────────────────────┤
│                                                              │
│   Developer commits secret                                   │
│          │                                                   │
│          ▼                                                   │
│   Push to remote ──────► Secret in Git history FOREVER       │
│          │                                                   │
│          ▼                                                   │
│   Bot scanners find it (minutes to hours)                   │
│          │                                                   │
│          ▼                                                   │
│   Attackers exploit credential                               │
│          │                                                   │
│          ▼                                                   │
│   Breach / Financial damage / Data loss                      │
│                                                              │
└─────────────────────────────────────────────────────────────┘

# This does NOT remove the secret from history
git rm config/secrets.yml
git commit -m "Remove secrets"
git push

# Anyone can still see it
git log --all --full-history -- config/secrets.yml
git show <commit-hash>:config/secrets.yml

# Install
brew install git-secrets  # macOS
# or
git clone https://github.com/awslabs/git-secrets.git
cd git-secrets && make install

# Initialize in repository
cd your-repo
git secrets --install

# Add AWS patterns (detects access keys, secret keys)
git secrets --register-aws

# Add custom patterns
git secrets --add 'private_key'
git secrets --add 'PRIVATE KEY'
git secrets --add --allowed 'AKIAEXAMPLE'  # Allow specific false positive

┌──────────────────────────────────────────────────────┐
│                    git commit                         │
│                        │                              │
│                        ▼                              │
│              ┌─────────────────┐                      │
│              │  pre-commit     │                      │
│              │  hook runs      │                      │
│              └────────┬────────┘                      │
│                       │                               │
│                       ▼                               │
│              ┌─────────────────┐                      │
│              │  git-secrets    │                      │
│              │  scans staged   │                      │
│              │  files          │                      │
│              └────────┬────────┘                      │
│                       │                               │
│          ┌────────────┴────────────┐                  │
│          │                         │                  │
│          ▼                         ▼                  │
│   ┌─────────────┐          ┌─────────────┐           │
│   │ No secrets  │          │ Secret found │           │
│   │ Commit OK   │          │ BLOCKED      │           │
│   └─────────────┘          └─────────────┘           │
└──────────────────────────────────────────────────────┘

# Install
brew install gitleaks  # macOS
# or download from https://github.com/gitleaks/gitleaks/releases

# Scan current state
gitleaks detect --source . --verbose

# Scan entire history
gitleaks detect --source . --verbose --log-opts="--all"

# Generate report
gitleaks detect --source . --report-format json --report-path report.json

[extend]
useDefault = true

[[rules]]
id = "custom-api-key"
description = "Custom API Key"
regex = '''(?i)mycompany[_-]?api[_-]?key\s*[=:]\s*['"]?([a-zA-Z0-9]{32,})['"]?'''
secretGroup = 1

[[rules]]
id = "internal-token"
description = "Internal Service Token"
regex = '''INT_TOKEN_[A-Z0-9]{16,}'''

[allowlist]
paths = [
    '''\.gitleaks\.toml$''',
    '''.*test.*''',
    '''.*_test\.go$''',
]

# .pre-commit-config.yaml
repos:
  - repo: https://github.com/gitleaks/gitleaks
    rev: v8.18.1
    hooks:
      - id: gitleaks

  - repo: https://github.com/awslabs/git-secrets
    rev: master
    hooks:
      - id: git-secrets

  - repo: https://github.com/Yelp/detect-secrets
    rev: v1.4.0
    hooks:
      - id: detect-secrets
        args: ['--baseline', '.secrets.baseline']

pip install pre-commit
pre-commit install
pre-commit run --all-files  # Test on existing files

# gitleaks - scan all branches and history
gitleaks detect \
  --source . \
  --log-opts="--all --full-history" \
  --report-format sarif \
  --report-path gitleaks-report.sarif

# truffleHog - entropy and pattern detection
pip install truffleHog
trufflehog git file://. --only-verified --json > trufflehog-report.json

# truffleHog for remote repos
trufflehog git https://github.com/org/repo --only-verified

name: Secret Scanning
on: [push, pull_request]

jobs:
  gitleaks:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Full history for scanning

      - name: Run gitleaks
        uses: gitleaks/gitleaks-action@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }}  # Optional for extra features

secret_detection:
  stage: test
  image: zricethezav/gitleaks:latest
  script:
    - gitleaks detect --source . --verbose --exit-code 1
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

□ 1. ROTATE THE SECRET IMMEDIATELY
     - Don't wait for history cleanup
     - Assume the secret is compromised

□ 2. AUDIT ACCESS LOGS
     - Check if the secret was used maliciously
     - Review cloud provider activity logs

□ 3. REMOVE FROM HISTORY
     - Use BFG or git filter-branch
     - Force push to all branches

□ 4. NOTIFY COLLABORATORS
     - They must re-clone or reset
     - Old clones still have the secret

□ 5. ENABLE PREVENTIVE MEASURES
     - Install pre-commit hooks
     - Enable GitHub secret scanning

# Download BFG
wget https://repo1.maven.org/maven2/com/madgag/bfg/1.14.0/bfg-1.14.0.jar

# Create file with secrets to remove
echo "AKIAIOSFODNN7EXAMPLE" > secrets-to-remove.txt
echo "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" >> secrets-to-remove.txt

# Clone a fresh mirror
git clone --mirror [email protected]:org/repo.git

# Run BFG
java -jar bfg-1.14.0.jar --replace-text secrets-to-remove.txt repo.git

# Clean up
cd repo.git
git reflog expire --expire=now --all
git gc --prune=now --aggressive

# Force push
git push --force

pip install git-filter-repo

# Replace secret with ***REMOVED***
git filter-repo --replace-text <(echo 'AKIAIOSFODNN7EXAMPLE==>***REMOVED***')

# Or use expressions file
cat > expressions.txt << EOF
AKIAIOSFODNN7EXAMPLE==>***REMOVED***
regex:password\s*=\s*['"][^'"]+['"]==>password='***REMOVED***'
EOF

git filter-repo --replace-text expressions.txt

# Remove specific file from all history
git filter-branch --force --index-filter \
  'git rm --cached --ignore-unmatch path/to/secret/file' \
  --prune-empty --tag-name-filter cat -- --all

# Clean up
git reflog expire --expire=now --all
git gc --prune=now --aggressive
git push origin --force --all
git push origin --force --tags

# All team members must either:

# Option 1: Fresh clone (safest)
rm -rf repo
git clone [email protected]:org/repo.git

# Option 2: Reset existing clone
git fetch origin
git reset --hard origin/main
git reflog expire --expire=now --all
git gc --prune=now --aggressive

┌─────────────────────────────────────────────────────────────┐
│                    SECRET HIERARCHY                          │
├─────────────────────────────────────────────────────────────┤
│                                                              │
│   PRODUCTION                                                 │
│   ├── AWS Secrets Manager / HashiCorp Vault                 │
│   ├── Injected at runtime via IAM roles                     │
│   └── Never in code or config files                         │
│                                                              │
│   CI/CD                                                      │
│   ├── GitHub Actions Secrets / GitLab CI Variables          │
│   ├── OIDC for cloud authentication (no static keys)        │
│   └── Scoped to environments (staging, production)          │
│                                                              │
│   LOCAL DEVELOPMENT                                          │
│   ├── .env files (in .gitignore)                           │
│   ├── .env.example with placeholders (committed)            │
│   └── Secret manager CLI tools (1Password, Vault)           │
│                                                              │
└─────────────────────────────────────────────────────────────┘

# Environment files
.env
.env.local
.env.*.local
.env.production
.env.development

# Credential files
credentials.json
service-account.json
*-credentials.json
*.pem
*.key
id_rsa
id_ed25519

# Cloud provider configs
.aws/credentials
.azure/credentials
gcloud-service-key.json

# IDE and tool configs that might contain secrets
.idea/**/dataSources/
.vscode/settings.json

# Terraform state (contains secrets)
*.tfstate
*.tfstate.*
.terraform/

# Kubernetes secrets
*-secret.yaml
*-secrets.yaml

# .env.example (committed)
DATABASE_URL=postgresql://user:password@localhost:5432/myapp
API_KEY=your-api-key-here
AWS_ACCESS_KEY_ID=your-access-key
AWS_SECRET_ACCESS_KEY=your-secret-key

# .env (in .gitignore - never committed)
DATABASE_URL=postgresql://admin:[email protected]:5432/myapp
API_KEY=sk_live_abc123...
AWS_ACCESS_KEY_ID=AKIA...
AWS_SECRET_ACCESS_KEY=wJalrXUtn...

# Example rotation schedule
secrets:
  api_keys:
    rotation: 90 days
    notification: 14 days before expiry

  database_passwords:
    rotation: 30 days
    automated: true  # Use AWS Secrets Manager rotation

  jwt_signing_keys:
    rotation: 180 days
    overlap_period: 24 hours  # Support old and new keys

  oauth_client_secrets:
    rotation: 365 days
    requires: Manual application update

# Set up template directory
mkdir -p ~/.git-templates/hooks
git secrets --install ~/.git-templates/hooks

# Configure git to use templates
git config --global init.templateDir ~/.git-templates

# Add patterns globally
git secrets --add --global 'PRIVATE KEY'
git secrets --add --global --allowed 'AKIAEXAMPLE'
git secrets --register-aws --global

# New repos automatically get hooks
git init new-project  # Has git-secrets hooks

# Create baseline with detect-secrets
pip install detect-secrets
detect-secrets scan > .secrets.baseline

# Audit the baseline
detect-secrets audit .secrets.baseline

# Future scans compare against baseline
detect-secrets scan --baseline .secrets.baseline

# GitHub Actions: Weekly full scan with notifications
name: Weekly Secret Audit
on:
  schedule:
    - cron: '0 9 * * 1'  # Monday 9 AM

jobs:
  audit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Run gitleaks
        id: gitleaks
        uses: gitleaks/gitleaks-action@v2
        continue-on-error: true

      - name: Notify on findings
        if: steps.gitleaks.outcome == 'failure'
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "⚠️ Secrets detected in ${{ github.repository }}"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

# Immediately scan for secrets
gitleaks detect --source . --verbose

# Check if specific string is in history
git log -S "AKIAIOSFODNN7EXAMPLE" --all

# Find commits that touched a file
git log --all --full-history -- path/to/secrets.env

# Quick removal (creates new history)
git filter-repo --path path/to/secrets.env --invert-paths