feat: refactoring

.flake8

@@ -0,0 +1,28 @@
+[flake8]
+max-line-length = 88
+extend-ignore = 
+    E203,
+    E501,
+    W503,
+    E402
+exclude = 
+    .git,
+    __pycache__,
+    .pytest_cache,
+    .mypy_cache,
+    .venv,
+    venv,
+    build,
+    dist,
+    outputs,
+    tmp,
+    tools,
+    res/phonics/pinyin-data
+per-file-ignores =
+    # Tests can have longer lines and different import patterns
+    tests/*:E501,F401,F811,F841
+    # Legacy files may have different patterns
+    src/legacy/*:E402,F401
+    # Refactored code can have complex functions during development
+    src/refactored/*:C901
+max-complexity = 12

.github/workflows/ci-pipeline.yml

@@ -0,0 +1,158 @@
+name: CI Pipeline
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+env:
+  DOCKER_BUILDKIT: 1
+  COMPOSE_DOCKER_CLI_BUILD: 1
+
+jobs:
+  test:
+    name: Run Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # Install project with development dependencies
+          pip install -e ".[dev]"
+          # Install additional CI-specific tools
+          pip install safety
+
+      - name: Run security checks
+        run: |
+          # Security vulnerability scan using bandit.yaml config
+          bandit -r src/ -f json -o bandit-report.json -c bandit.yaml || echo "Security issues found"
+          # Dependency security check
+          safety check --output json > safety-report.json || echo "Dependency vulnerabilities found"
+
+      - name: Run unit tests with coverage
+        run: |
+          PYTHONPATH=src pytest tests/unit/ --cov=src --cov-report=xml --cov-report=html -v
+
+      - name: Run security tests
+        run: |
+          PYTHONPATH=src pytest tests/security/ -v
+
+      - name: Upload coverage reports
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella
+
+  docker-build:
+    name: Docker Build Test
+    runs-on: ubuntu-latest
+    needs: test
+
+    strategy:
+      matrix:
+        font-style: [han_serif, handwritten]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        run: |
+          cd docker
+          docker compose build mengshen-font
+
+      - name: Verify Docker image
+        run: |
+          cd docker
+          docker compose run --rm mengshen-font otfccdump --version
+          docker compose run --rm mengshen-font python --version
+
+      - name: Test template JSON generation
+        run: |
+          cd docker
+          docker compose run --rm -v $(pwd)/../outputs:/app/outputs -v $(pwd)/../tmp:/app/tmp mengshen-font \
+            bash -c "
+              set -e
+              echo 'Testing template JSON generation for ${{ matrix.font-style }}'
+              PYTHONPATH=src python -m refactored.scripts.make_template_jsons --style ${{ matrix.font-style }}
+              ls -la /app/tmp/json/
+            "
+
+      - name: Test Latin alphabet extraction
+        run: |
+          cd docker
+          docker compose run --rm -v $(pwd)/../outputs:/app/outputs -v $(pwd)/../tmp:/app/tmp mengshen-font \
+            bash -c "
+              set -e
+              echo 'Testing Latin alphabet extraction for ${{ matrix.font-style }}'
+              PYTHONPATH=src python -m refactored.scripts.retrieve_latin_alphabet --style ${{ matrix.font-style }}
+              ls -la /app/tmp/json/
+            "
+
+      - name: Test pattern table generation
+        run: |
+          cd docker
+          docker compose run --rm -v $(pwd)/../outputs:/app/outputs -v $(pwd)/../res:/app/res mengshen-font \
+            bash -c "
+              set -e
+              echo 'Testing pattern table generation'
+              cd res/phonics/duo_yin_zi/scripts && python make_pattern_table.py
+              cd ../../unicode_mapping_table && python make_unicode_pinyin_map_table.py
+              ls -la /app/outputs/
+            "
+
+      - name: Test dry-run font generation
+        run: |
+          cd docker
+          docker compose run --rm -v $(pwd)/../outputs:/app/outputs -v $(pwd)/../tmp:/app/tmp mengshen-font \
+            bash -c "
+              set -e
+              echo 'Testing dry-run font generation for ${{ matrix.font-style }}'
+              PYTHONPATH=src python -m refactored.cli.main -t ${{ matrix.font-style }} --dry-run --verbose
+            "
+
+
+  performance-test:
+    name: Performance Benchmark
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.event_name == 'pull_request'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # Install project with development dependencies
+          pip install -e ".[dev]"
+
+      - name: Run performance benchmarks
+        run: |
+          PYTHONPATH=src pytest tests/performance/ --benchmark-only --benchmark-json=benchmark.json -v
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: benchmark.json

.github/workflows/code-quality.yml

@@ -0,0 +1,122 @@
+name: Code Quality and Linting
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  lint:
+    name: Code Quality Checks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # Install project with development dependencies
+          pip install -e ".[dev]"
+          # Install additional CI-specific tools
+          pip install safety
+
+      - name: Check code formatting with Black
+        run: |
+          black --check --diff src/ tests/
+
+      - name: Check import sorting with isort
+        run: |
+          isort --check-only --diff src/ tests/
+
+      - name: Lint with flake8
+        run: |
+          flake8 src/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 src/ tests/ --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics
+
+      - name: Type checking with mypy
+        run: |
+          mypy src/refactored/ --ignore-missing-imports --no-strict-optional --disable-error-code union-attr --disable-error-code index --disable-error-code assignment --disable-error-code arg-type --disable-error-code return-value --disable-error-code call-overload --disable-error-code operator --disable-error-code attr-defined --disable-error-code misc --disable-error-code dict-item --disable-error-code import-untyped || echo "Type checking completed with warnings"
+
+      - name: Security analysis with bandit
+        run: |
+          # Use bandit.yaml config file for exclusions and skips
+          echo "Running bandit security analysis..."
+          bandit -r src/refactored/ -f json -o bandit-report.json -c bandit.yaml || BANDIT_EXIT_CODE=$?
+
+          # Display bandit report
+          echo "=== Bandit Security Report ==="
+          if [ -f bandit-report.json ]; then
+            echo "Security issues found:"
+            cat bandit-report.json | jq -r '.results[] | "File: \(.filename), Line: \(.line_number), Issue: \(.issue_text), Severity: \(.issue_severity)"' 2>/dev/null || cat bandit-report.json
+          fi
+
+          # Run bandit in text mode for human-readable output
+          echo "=== Detailed Bandit Output ==="
+          bandit -r src/refactored/ --severity-level medium -c bandit.yaml -f txt || BANDIT_TEXT_EXIT_CODE=$?
+
+          # Fail the step if bandit found issues
+          if [ "${BANDIT_EXIT_CODE:-0}" -ne 0 ]; then
+            echo "Bandit found security issues (exit code: ${BANDIT_EXIT_CODE:-0})"
+            exit 1
+          fi
+
+      - name: Dependency security check
+        run: |
+          safety check --output json > safety-report.json || echo "Dependency vulnerabilities found"
+
+      - name: Upload security reports
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: security-reports
+          path: |
+            bandit-report.json
+            safety-report.json
+
+  spellcheck:
+    name: Spell Check
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js for cspell
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+
+      - name: Install cspell
+        run: npm install -g cspell
+
+      - name: Run spell check
+        run: |
+          cspell "src/refactored/**/*.py" "tests/**/*.py" "*.md" "doc/*.md" --config cspell.json
+
+  markdown-lint:
+    name: Markdown Lint
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js for markdownlint
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+
+      - name: Install markdownlint-cli
+        run: npm install -g markdownlint-cli
+
+      - name: Lint Markdown files
+        run: |
+          markdownlint "*.md" "doc/*.md" --config .markdownlint.json --ignore node_modules