Update to encoding_rs, rustler 0.37, add dirty schedulers

.github/workflows/ci.yml

@@ -0,0 +1,110 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  MIX_ENV: test
+  RUSTLER_PRECOMPILATION_EXCODING_BUILD: "true"
+
+jobs:
+  test:
+    name: Test (Elixir ${{ matrix.elixir }} / OTP ${{ matrix.otp }})
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - elixir: "1.15"
+            otp: "26"
+          - elixir: "1.16"
+            otp: "26"
+          - elixir: "1.17"
+            otp: "27"
+          - elixir: "1.18"
+            otp: "27"
+          - elixir: "1.19"
+            otp: "28"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: ${{ matrix.elixir }}
+          otp-version: ${{ matrix.otp }}
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache deps
+        uses: actions/cache@v4
+        with:
+          path: deps
+          key: deps-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: deps-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-
+
+      - name: Cache _build
+        uses: actions/cache@v4
+        with:
+          path: _build
+          key: build-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: build-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-
+
+      - name: Cache Rust target
+        uses: actions/cache@v4
+        with:
+          path: native/excoding/target
+          key: rust-${{ runner.os }}-${{ hashFiles('native/excoding/Cargo.lock') }}
+          restore-keys: rust-${{ runner.os }}-
+
+      - name: Install dependencies
+        run: mix deps.get
+
+      - name: Compile
+        run: mix compile --warnings-as-errors
+
+      - name: Run tests
+        run: mix test
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    env:
+      RUSTLER_PRECOMPILATION_EXCODING_BUILD: "true"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: "1.19"
+          otp-version: "28"
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - name: Cache deps
+        uses: actions/cache@v4
+        with:
+          path: deps
+          key: deps-lint-${{ runner.os }}-${{ hashFiles('**/mix.lock') }}
+
+      - name: Install dependencies
+        run: mix deps.get
+
+      - name: Check formatting (Elixir)
+        run: mix format --check-formatted
+
+      - name: Check formatting (Rust)
+        run: cargo fmt --manifest-path native/excoding/Cargo.toml -- --check
+
+      - name: Clippy (Rust)
+        run: cargo clippy --manifest-path native/excoding/Cargo.toml -- -D warnings

.github/workflows/rustler_precompiled.yml

@@ -1,9 +1,18 @@
 name: Build precompiled NIFs
 
 on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to build (e.g., 0.2.1) - must match mix.exs'
+        required: true
+        type: string
   push:
     tags:
-      - '*'
+      - 'v*'
+
+permissions:
+  contents: write
 
 jobs:
   build_release:
@@ -12,54 +21,70 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        nif: ["2.16", "2.15"]
+        nif: ["2.17", "2.16", "2.15"]
         job:
-          - { target: arm-unknown-linux-gnueabihf , os: ubuntu-20.04 , use-cross: true }
-          - { target: aarch64-unknown-linux-gnu   , os: ubuntu-20.04 , use-cross: true }
-          - { target: aarch64-unknown-linux-musl  , os: ubuntu-20.04 , use-cross: true }
-          - { target: aarch64-apple-darwin        , os: macos-11      }
-          - { target: riscv64gc-unknown-linux-gnu , os: ubuntu-20.04 , use-cross: true }
-          - { target: x86_64-apple-darwin         , os: macos-11      }
-          - { target: x86_64-unknown-linux-gnu    , os: ubuntu-20.04  }
-          - { target: x86_64-unknown-linux-musl   , os: ubuntu-20.04 , use-cross: true }
-          - { target: x86_64-pc-windows-gnu       , os: windows-2019  }
-          - { target: x86_64-pc-windows-msvc      , os: windows-2019  }
+          - { target: arm-unknown-linux-gnueabihf , os: ubuntu-22.04 , use-cross: true }
+          - { target: aarch64-unknown-linux-gnu   , os: ubuntu-22.04 , use-cross: true }
+          - { target: aarch64-unknown-linux-musl  , os: ubuntu-22.04 , use-cross: true }
+          - { target: aarch64-apple-darwin        , os: macos-14     }
+          - { target: riscv64gc-unknown-linux-gnu , os: ubuntu-22.04 , use-cross: true }
+          - { target: x86_64-apple-darwin         , os: macos-15     }
+          - { target: x86_64-unknown-linux-gnu    , os: ubuntu-22.04 }
+          - { target: x86_64-unknown-linux-musl   , os: ubuntu-22.04 , use-cross: true }
+          - { target: x86_64-pc-windows-gnu       , os: windows-2022 }
+          - { target: x86_64-pc-windows-msvc      , os: windows-2022 }
 
     steps:
-    - name: Checkout source code
-      uses: actions/checkout@v3
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Set project version
+        shell: bash
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "PROJECT_VERSION=${{ inputs.version }}" >> $GITHUB_ENV
+          else
+            # Extract from tag (v0.2.1 -> 0.2.1)
+            echo "PROJECT_VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+          fi
+
+      - name: Verify version matches mix.exs
+        shell: bash
+        run: |
+          MIX_VERSION=$(sed -n 's/^  @version "\(.*\)"/\1/p' mix.exs | head -n1)
+          if [ "$MIX_VERSION" != "$PROJECT_VERSION" ]; then
+            echo "::error::Version mismatch! Input: $PROJECT_VERSION, mix.exs: $MIX_VERSION"
+            exit 1
+          fi
+          echo "Building version $PROJECT_VERSION"
 
-    - name: Extract project version
-      shell: bash
-      run: |
-        # Get the project version from mix.exs
-        echo "PROJECT_VERSION=$(sed -n 's/^  @version "\(.*\)"/\1/p' mix.exs | head -n1)" >> $GITHUB_ENV
-    - name: Install Rust toolchain
-      uses: dtolnay/rust-toolchain@stable
-      with:
-        toolchain: stable
-        target: ${{ matrix.job.target }}
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          targets: ${{ matrix.job.target }}
 
-    - name: Build the project
-      id: build-crate
-      uses: philss/rustler-precompiled-action@v1.0.0
-      with:
-        project-name: excoding
-        project-version: ${{ env.PROJECT_VERSION }}
-        target: ${{ matrix.job.target }}
-        nif-version: ${{ matrix.nif }}
-        use-cross: ${{ matrix.job.use-cross }}
-        project-dir: "native/excoding"
+      - name: Build the project
+        id: build-crate
+        uses: philss/rustler-precompiled-action@v1.1.4
+        with:
+          project-name: excoding
+          project-version: ${{ env.PROJECT_VERSION }}
+          target: ${{ matrix.job.target }}
+          nif-version: ${{ matrix.nif }}
+          use-cross: ${{ matrix.job.use-cross }}
+          project-dir: "native/excoding"
 
-    - name: Artifact upload
-      uses: actions/upload-artifact@v3
-      with:
-        name: ${{ steps.build-crate.outputs.file-name }}
-        path: ${{ steps.build-crate.outputs.file-path }}
+      - name: Artifact upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.build-crate.outputs.file-name }}
+          path: ${{ steps.build-crate.outputs.file-path }}
 
-    - name: Publish archives and packages
-      uses: softprops/action-gh-release@v1
-      with:
-        files: |
-          ${{ steps.build-crate.outputs.file-path }}
-      if: startsWith(github.ref, 'refs/tags/')
+      - name: Publish to GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ env.PROJECT_VERSION }}
+          draft: ${{ github.event_name == 'workflow_dispatch' }}
+          files: |
+            ${{ steps.build-crate.outputs.file-path }}

.gitignore

@@ -29,4 +29,5 @@ excoding-*.tar
 
 /native/*/target
 
-checksum-*
+# Local notes (not committed)
+*.local.md

CHANGELOG.md

@@ -0,0 +1,49 @@
+# Changelog
+
+## v0.2.1 (2025-01-22)
+
+### New Features
+
+- **Streaming decoder** (`Excoding.Decoder`): Stateful decoder for chunked/streaming data that properly handles multibyte characters split across chunk boundaries. Essential for processing file streams or network data in encodings like Shift_JIS, GBK, Big5, etc.
+  - `Excoding.Decoder.new/1` - Create a stateful decoder
+  - `Excoding.Decoder.decode_chunk/3` - Decode a chunk with state preservation
+  - `Excoding.Decoder.stream/2` - Stream transformer for use with `File.stream!/3`
+  - `Excoding.Decoder.stream_with_errors/2` - Stream transformer with error tracking
+- **BOM detection**: Detect encoding from Byte Order Marks (UTF-8, UTF-16LE, UTF-16BE)
+  - `detect_bom/1` - Detect BOM and return encoding name and BOM length
+  - `detect_and_strip_bom/1` - Detect BOM, strip it, and return encoding with remaining data
+
+## v0.2.0 (2025-01-22)
+
+### Breaking Changes
+
+- `encode/2` now returns `{:ok, binary}` or `{:error, reason}` (previously returned raw binary)
+- `decode/2` now returns `{:ok, string}` or `{:error, reason}` (previously returned raw string)
+- Use `encode!/2` and `decode!/2` for the old behavior (returns raw value, raises on error)
+
+### New Features
+
+- **Switched to `encoding_rs`**: Now uses the same encoding library as Firefox for better performance and active maintenance
+- **Dirty schedulers**: Operations on binaries larger than 64KB automatically use dirty CPU schedulers to avoid blocking the BEAM
+- **New functions**:
+  - `encode!/2` - Encode, returns raw value or raises
+  - `decode!/2` - Decode, returns raw value or raises
+  - `encoding_exists?/1` - Check if an encoding is supported
+  - `canonical_name/1` - Get the canonical WHATWG name for an encoding alias
+  - `list_encodings/0` - List all supported encodings
+  - `dirty_threshold/0` - Get the threshold for dirty scheduler usage
+
+### Improvements
+
+- Updated to Rust 2021 edition
+- Updated to rustler 0.37
+- Updated to rustler_precompiled 0.8
+- Removed unused dependencies (lazy_static, rustler_codegen)
+- Improved documentation with examples
+- Better error handling - no more panics in Rust code
+- Fixes OTP-26+ compilation issues (#40)
+- Fixes panic on error (#24)
+
+## v0.1.5 and earlier
+
+See [GitHub releases](https://github.com/elixir-ecto/excoding/releases) for previous versions.