Merge pull request #554 from LedgerHQ/cev/B2CA-901_improve-ragger-tests

B2CA-901: improve ragger tests

.github/workflows/build_and_functional_tests.yml

@@ -23,7 +23,7 @@ jobs:
     uses: LedgerHQ/ledger-app-workflows/.github/workflows/reusable_build.yml@v1
     with:
       upload_app_binaries_artifact: "ragger_elfs"
-      flags: "DEBUG=1 CAL_TEST_KEY=1 DOMAIN_NAME_TEST_KEY=1 SET_PLUGIN_TEST_KEY=1 NFT_TEST_KEY=1"
+      flags: "CAL_TEST_KEY=1 DOMAIN_NAME_TEST_KEY=1 SET_PLUGIN_TEST_KEY=1 NFT_TEST_KEY=1"
 
   ragger_tests:
     name: Run ragger tests using the reusable workflow
@@ -31,3 +31,22 @@ jobs:
     uses: LedgerHQ/ledger-app-workflows/.github/workflows/reusable_ragger_tests.yml@v1
     with:
       download_app_binaries_artifact: "ragger_elfs"
+
+  build_clone_app:
+    name: Build Clone app using the reusable workflow
+    uses: LedgerHQ/ledger-app-workflows/.github/workflows/reusable_build.yml@v1
+    with:
+      flags: "CHAIN=thundercore"
+      upload_app_binaries_artifact: "clone_elfs"
+
+  ragger_clone_tests:
+    name: Run ragger Clone tests using the reusable workflow
+    needs:
+      - build_application
+      - build_clone_app
+    uses: LedgerHQ/ledger-app-workflows/.github/workflows/reusable_ragger_tests.yml@v1
+    with:
+      download_app_binaries_artifact: "ragger_elfs"
+      additional_app_binaries_artifact: "clone_elfs"
+      additional_app_binaries_artifact_dir: ./tests/ragger/.test_dependencies/clone/build/
+      test_options: "--with_lib_mode"

.github/workflows/ci-workflow.yml

@@ -1,137 +0,0 @@
----
-name: Tests (Speculos and Zemu)
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - master
-      - develop
-  pull_request:
-
-jobs:
-
-  # =====================================================
-  #            ZEMU TESTS
-  # =====================================================
-
-  building_for_e2e_zemu_tests:
-    name: Building binaries for E2E Zemu tests
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder-lite:latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Build testing binaries
-        run: |
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          cd tests/zemu/ && ./build_local_test_elfs.sh
-
-      - name: Upload app binaries
-        uses: actions/upload-artifact@v4
-        with:
-          name: e2e_zemu_elfs
-          path: ./tests/zemu/elfs/
-
-  jobs-e2e-zemu-tests:
-    name: E2E Zemu tests
-    needs: [building_for_e2e_zemu_tests]
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Install additional tools
-        run: sudo apt-get update -y && sudo apt-get install -y libusb-1.0.0 libudev-dev
-
-      - name: Install NodeJS
-        uses: actions/setup-node@v3
-        with:
-          node-version: "16"
-
-      - name: Install yarn
-        run: npm install -g yarn
-
-      - name: Build/Install build js deps
-        run: cd tests/zemu/ && yarn install
-
-      - name: Create tmp folder for artifacts
-        run: mkdir tests/zemu/elfs
-
-      - name: Download app binaries
-        uses: actions/download-artifact@v4
-        with:
-          path: tmp/
-
-      - name: Gather elfs
-        run: cp tmp/e2e_zemu_elfs/*.elf tests/zemu/elfs/
-
-      - name: Run zemu tests
-        run: cd tests/zemu/ && yarn test
-
-  # =====================================================
-  #            SPECULOS TESTS
-  # =====================================================
-
-
-  building_for_e2e_speculos_tests:
-    name: Building binaries for E2E Speculos tests
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder-lite:latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Build testing binaries
-        run: |
-          mkdir tests/speculos/elfs
-          make clean && make -j DEBUG=1 NFT_STAGING_KEY=1 BOLOS_SDK=$NANOS_SDK && mv bin/app.elf tests/speculos/elfs/nanos.elf
-          make clean && make -j DEBUG=1 NFT_STAGING_KEY=1 BOLOS_SDK=$NANOX_SDK && mv bin/app.elf tests/speculos/elfs/nanox.elf
-          make clean && make -j DEBUG=1 NFT_STAGING_KEY=1 BOLOS_SDK=$NANOSP_SDK && mv bin/app.elf tests/speculos/elfs/nanosp.elf
-
-      - name: Upload app binaries
-        uses: actions/upload-artifact@v4
-        with:
-          name: e2e_speculos_elfs
-          path: ./tests/speculos/elfs
-
-
-  jobs-e2e-speculos-tests:
-    name: Speculos tests
-    strategy:
-      fail-fast: false
-      matrix:
-        model: ["nanosp", "nanos", "nanox"]
-
-    needs: [building_for_e2e_speculos_tests]
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v4
-
-      - name: Create tmp folder for artifacts
-        run: mkdir tests/speculos/elfs
-
-      - name: Download app binaries
-        uses: actions/download-artifact@v4
-        with:
-          path: tmp/
-
-      - name: Gather elfs
-        run: cp tmp/e2e_speculos_elfs/*.elf tests/speculos/elfs/
-
-      - name: Install dependencies
-        run: |
-          cd tests/speculos
-          sudo apt-get update && sudo apt-get install -y qemu-user-static
-          pip install -r requirements.txt
-
-      - name: Run speculos tests
-        run: |
-          cd tests/speculos
-          pytest --model ${{ matrix.model }} --path ./elfs/${{ matrix.model }}.elf --display headless

README.md

@@ -25,16 +25,15 @@
 - [About the project](#about-the-project)
 - [Documentation](#documentation)
   - [Plugins](#plugins)
-- [Testing](#testing)
-  - [Requirements](#requirements)
-    - [Build the applications required by the test suite](#build-the-applications-required-by-the-test-suite)
-  - [Running all tests](#running-all-tests)
-    - [With Makefile](#with-makefile)
-    - [With yarn](#with-yarn)
-  - [Running a specific test](#running-a-specific-test)
-  - [Adding tests](#adding-tests)
-    - [Zemu](#zemu)
-    - [Update binaries](#update-binaries)
+- [Quick start guide](#quick-start-guide)
+  - [With VSCode](#with-vscode)
+  - [With a terminal](#with-a-terminal)
+- [Compilation and load](#compilation-and-load)
+  - [Compilation](#compilation)
+  - [Loading on a physical device](#loading-on-a-physical-device)
+- [Tests](#tests)
+  - [Functional Tests (Ragger based)](#functional-tests-ragger-based)
+  - [Unit Tests](#unit-tests)
 - [Contributing](#contributing)
 
 </details>
@@ -60,94 +59,201 @@ Find the documentations here:
 - [Plugin guide](https://hackmd.io/300Ukv5gSbCbVcp3cZuwRQ)
 - [Boilerplate plugin](https://github.com/LedgerHQ/app-plugin-boilerplate)
 
-## Testing
+## Quick start guide
 
-Testing is done via the open-source framework [zemu](https://github.com/Zondax/zemu).
+### With VSCode
 
-### Requirements
+You can quickly setup a convenient environment to build and test your application by using
+[Ledger's VSCode developer tools extension](https://marketplace.visualstudio.com/items?itemName=LedgerHQ.ledger-dev-tools)
+which leverages the [ledger-app-dev-tools](https://github.com/LedgerHQ/ledger-app-builder/pkgs/container/ledger-app-builder%2Fledger-app-dev-tools)
+docker image.
 
-- [nodeJS == 16](https://github.com/nvm-sh/nvm)
-- [yarn](https://classic.yarnpkg.com/lang/en/docs/install/#debian-stable)
-- [build environment](https://github.com/LedgerHQ/ledger-app-builder/blob/master/Dockerfile)
+It will allow you, whether you are developing on macOS, Windows or Linux,
+to quickly **build** your apps, **test** them on **Speculos** and **load** them on any supported device.
 
-#### Build the applications required by the test suite
+- Install and run [Docker](https://www.docker.com/products/docker-desktop/).
+- Make sure you have an X11 server running:
+  - On Ubuntu Linux, it should be running by default.
+  - On macOS, install and launch [XQuartz](https://www.xquartz.org/)
+    (make sure to go to XQuartz > Preferences > Security and check "Allow client connections").
+  - On Windows, install and launch [VcXsrv](https://sourceforge.net/projects/vcxsrv/)
+    (make sure to configure it to disable access control).
+- Install [VScode](https://code.visualstudio.com/download) and add [Ledger's extension](https://marketplace.visualstudio.com/items?itemName=LedgerHQ.ledger-dev-tools).
+- Open a terminal and clone `app-ethereum` with `git clone git@github.com:LedgerHQ/app-ethereum.git`.
+- Open the `app-ethereum` folder with VSCode.
+- Use Ledger extension's sidebar menu or open the tasks menu with `ctrl + shift + b`
+  (`command + shift + b` on a Mac) to conveniently execute actions:
+  - Build the app for the device model of your choice with `Build`.
+  - Test your binary on [Speculos](https://github.com/LedgerHQ/speculos) with `Run with Speculos`.
+  - You can also run functional tests, load the app on a physical device, and more.
 
-1. Add your BOLOS SDKs path to:
+> The terminal tab of VSCode will show you what commands the extension runs behind the scene.
 
-    `NANOS_SDK` and `NANOX_SDK`
+### With a terminal
 
-2. Go to the `tests` folder and run `./build_local_test_elfs.sh`
+The [ledger-app-dev-tools](https://github.com/LedgerHQ/ledger-app-builder/pkgs/container/ledger-app-builder%2Fledger-app-dev-tools)
+docker image contains all the required tools and libraries to **build**, **test** and **load** an application.
 
-    ```sh
-    cd tests
-    # This helper script will build the applications required by the test suite and move them at the right place.
-    yarn install
-    ./build_local_test_elfs.sh
-    ```
+You can download it from the ghcr.io docker repository:
 
-### Running all tests
-
-#### With Makefile
-
-1. Then you can install and run tests by simply running on the `root` of the repo:
-
-    ```sh
-    make test
-    # This will run `make install_tests` and `make run_tests`
-    ```
-
-#### With yarn
-
-1. Go to the `tests` folder and run:
-
-    ```sh
-    yarn test
-    ```
-
-### Running a specific test
-
-1. Go to the `tests` folder and run:
-
-    ```sh
-    yarn jest --runInBand --detectOpenHandles {YourTestFile}
-    ```
-
-2. For example with the `send test`:
-
-    ```sh
-    yarn jest --runInBand --detectOpenHandles src/send.test.js
-    ```
-
-### Adding tests
-
-#### Zemu
-
-To add tests, copy one of the already existing test files in `tests/src/`.
-You then need to adapt the `buffer` and `tx` variables to adapt to the APDU you wish to send.
-
-- Adapt the expected screen flow. Please create a folder under `tests/snapshots` with the name of the test you're performing.
-- Then adapt the `ORIGINAL_SNAPSHOT_PATH_PREFIX` with the name of the folder you just created.
-- To create the snapshots, modify the `SNAPSHOT_PATH_PREFIX` and set it to be equal to `ORIGINAL_SNAPSHOT_PATH_PREFIX`.
-- Run the tests once, this will create all the snapshots in the folder you created.
-- Put back your `SNAPSHOT_PATH_PREFIX` to `snapshots/tmp/`.
-
-Finally make sure you adapt the expected signature!
-
-#### Update binaries
-
-Don't forget to update the binaries in the test folder. To do so, compile with those environment variables:
-
-```sh
-make DEBUG=1 ALLOW_DATA=1
+```shell
+sudo docker pull ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest
 ```
 
-Then copy the binary to the `tests/elfs` folder (in this case, compiled with SDK for nanoS):
+You can then enter this development environment by executing the following command
+from the root directory of the application `git` repository:
 
-```sh
-cp bin/app.elf tests/elfs/ethereum_nanos.elf
+#### Linux (Ubuntu)
+
+```shell
+sudo docker run --rm -ti --user "$(id -u):$(id -g)" --privileged -v "/dev/bus/usb:/dev/bus/usb" -v "$(realpath .):/app" ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest
 ```
 
-Repeat the operation for a binary compiled with nanoX SDK and change for `ethereum_nanox.elf`.
+#### macOS
+
+```shell
+sudo docker run  --rm -ti --user "$(id -u):$(id -g)" --privileged -v "$(pwd -P):/app" ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest
+```
+
+#### Windows (with PowerShell)
+
+```shell
+docker run --rm -ti --privileged -v "$(Get-Location):/app" ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest
+```
+
+The application's code will be available from inside the docker container,
+you can proceed to the following compilation steps to build your app.
+
+## Compilation and load
+
+To easily setup a development environment for compilation and loading on a physical device, you can use the [VSCode integration](#with-vscode)
+whether you are on Linux, macOS or Windows.
+
+If you prefer using a terminal to perform the steps manually, you can use the guide below.
+
+### Compilation
+
+Setup a compilation environment by following the [shell with docker approach](#with-a-terminal).
+
+From inside the container, use the following command to build the app:
+
+```shell
+make DEBUG=1  # compile optionally with PRINTF
+```
+
+You can choose which device to compile and load for by setting the `BOLOS_SDK` environment variable to the following values:
+
+- `BOLOS_SDK=$NANOS_SDK`
+- `BOLOS_SDK=$NANOX_SDK`
+- `BOLOS_SDK=$NANOSP_SDK`
+- `BOLOS_SDK=$STAX_SDK`
+
+### Loading on a physical device
+
+This step will vary slightly depending on your platform.
+
+> Your physical device must be connected, unlocked and the screen showing the dashboard (not inside an application).
+
+#### Linux (Ubuntu)
+
+First make sure you have the proper udev rules added on your host.
+See [udev-rules](https://github.com/LedgerHQ/udev-rules)
+
+Then once you have [opened a terminal](#with-a-terminal) in the `app-builder` image and [built the app](#compilation-and-load)
+for the device you want, run the following command:
+
+```shell
+# Run this command from the app-builder container terminal.
+make load    # load the app on a Nano S by default
+```
+
+[Setting the BOLOS_SDK environment variable](#compilation-and-load) will allow you to load
+on whichever supported device you want.
+
+#### macOS / Windows (with PowerShell)
+
+> It is assumed you have [Python](https://www.python.org/downloads/) installed on your computer.
+
+Run these commands on your host from the app's source folder once you have [built the app](#compilation-and-load)
+for the device you want:
+
+```shell
+# Install Python virtualenv
+python3 -m pip install virtualenv
+# Create the 'ledger' virtualenv
+python3 -m virtualenv ledger
+```
+
+Enter the Python virtual environment
+
+- macOS: `source ledger/bin/activate`
+- Windows: `.\ledger\Scripts\Activate.ps1`
+
+```shell
+# Install Ledgerblue (tool to load the app)
+python3 -m pip install ledgerblue
+# Load the app.
+python3 -m ledgerblue.runScript --scp --fileName bin/app.apdu --elfFile bin/app.elf
+```
+
+## Tests
+
+The Ethereum app comes with different tests:
+
+- Functional Tests implemented with Ledger's [Ragger](https://github.com/LedgerHQ/ragger) test framework.
+- Unit Tests, allowing to test basic simple functions
+
+### Functional Tests (Ragger based)
+
+#### Linux (Ubuntu)
+
+On Linux, you can use [Ledger's VS Code extension](#with-vscode) to run the tests.
+If you prefer not to, open a terminal and follow the steps below.
+
+Install the tests requirements:
+
+```shell
+pip install -r tests/ragger/requirements.txt
+```
+
+Then you can:
+
+Run the functional tests (here for nanos but available for any device once you have built the binaries):
+
+```shell
+pytest tests/ragger/ --tb=short -v --device nanos
+```
+
+Please see the corresponding ducomentation [USAGE](tests/ragger/usage.md)
+
+Or run your app directly with Speculos
+
+```shell
+speculos --model nanos build/nanos/bin/app.elf
+```
+
+#### macOS / Windows
+
+To test your app on macOS or Windows, it is recommended to use [Ledger's VS Code extension](#with-vscode)
+to quickly setup a working test environment.
+
+You can use the following sequence of tasks and commands (all accessible in the **extension sidebar menu**):
+
+- `Select build target`
+- `Build app`
+
+Then you can choose to execute the functional tests:
+
+- Use `Run tests`.
+
+Or simply run the app on the Speculos emulator:
+
+- `Run with Speculos`.
+
+### Unit Tests
+
+Those tests are available in the directory `tests/unit`. Please see the corresponding [README](tests/unit/README.md)
+to compile and run them.
 
 ## Contributing
 

client/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.1] - 2024-04-05
+
+### Added
+
+- Add new function `send_raw`, allowing to send a raw payload APDU
+- Add new error code definition
+
 ## [0.4.0] - 2024-04-03
 
 ### Added

client/src/ledger_app_clients/ethereum/__init__.py

@@ -1,4 +1,4 @@
 try:
-    from ledger_app_clients.ethereum.__version__ import __version__  # noqa
+    from __version__ import __version__  # noqa
 except ImportError:
     __version__ = "unknown version"  # noqa

client/src/ledger_app_clients/ethereum/client.py

@@ -21,6 +21,7 @@ class StatusWord(IntEnum):
     INVALID_P1_P2 = 0x6b00
     CONDITION_NOT_SATISFIED = 0x6985
     REF_DATA_NOT_FOUND = 0x6a88
+    EXCEPTION_OVERFLOW = 0x6807
 
 
 class DomainNameTag(IntEnum):
@@ -49,6 +50,15 @@ class EthAppClient:
     def response(self) -> Optional[RAPDU]:
         return self._client.last_async_response
 
+    def send_raw(self, cla: int, ins: int, p1: int, p2: int, payload: bytes):
+        header = bytearray()
+        header.append(cla)
+        header.append(ins)
+        header.append(p1)
+        header.append(p2)
+        header.append(len(payload))
+        return self._exchange(header + payload)
+
     def eip712_send_struct_def_struct_name(self, name: str):
         return self._exchange_async(self._cmd_builder.eip712_send_struct_def_struct_name(name))
 
@@ -131,6 +141,12 @@ class EthAppClient:
                                                                       bip32_path,
                                                                       chain_id))
 
+    def get_eth2_public_addr(self,
+                             display: bool = True,
+                             bip32_path: str = "m/12381/3600/0/0"):
+        return self._exchange_async(self._cmd_builder.get_eth2_public_addr(display,
+                                                                           bip32_path))
+
     def provide_domain_name(self, challenge: int, name: str, addr: bytes) -> RAPDU:
         payload = format_tlv(DomainNameTag.STRUCTURE_TYPE, 3)  # TrustedDomainName
         payload += format_tlv(DomainNameTag.STRUCTURE_VERSION, 1)

client/src/ledger_app_clients/ethereum/command_builder.py

@@ -11,6 +11,7 @@ from .eip712 import EIP712FieldType
 
 class InsType(IntEnum):
     GET_PUBLIC_ADDR = 0x02
+    GET_ETH2_PUBLIC_ADDR = 0x0e
     SIGN = 0x04
     PERSONAL_SIGN = 0x08
     PROVIDE_ERC20_TOKEN_INFORMATION = 0x0a
@@ -250,6 +251,15 @@ class CommandBuilder:
                                int(chaincode),
                                payload)
 
+    def get_eth2_public_addr(self,
+                             display: bool,
+                             bip32_path: str) -> bytes:
+        payload = pack_derivation_path(bip32_path)
+        return self._serialize(InsType.GET_ETH2_PUBLIC_ADDR,
+                               int(display),
+                               0x00,
+                               payload)
+
     def set_plugin(self,
                    type_: int,
                    version: int,

client/src/ledger_app_clients/ethereum/eip712/InputData.py

@@ -6,8 +6,8 @@ import sys
 import copy
 from typing import Any, Callable, Optional
 
-from ledger_app_clients.ethereum import keychain
-from ledger_app_clients.ethereum.client import EthAppClient, EIP712FieldType
+from client import keychain
+from client.client import EthAppClient, EIP712FieldType
 
 
 # global variables

tests/ragger/client

@@ -1 +1 @@
-../../client
+../../client/src/ledger_app_clients/ethereum

tests/ragger/conftest.py

@@ -1,4 +1,49 @@
+import sys
+from pathlib import Path
+import warnings
+import glob
+
 from ragger.conftest import configuration
 
+#######################
+# CONFIGURATION START #
+#######################
+
+# You can configure optional parameters by overriding the value of
+# ragger.configuration.OPTIONAL_CONFIGURATION
+# Please refer to ragger/conftest/configuration.py for their descriptions and accepted values
+
+def pytest_addoption(parser):
+    parser.addoption("--with_lib_mode", action="store_true", help="Run the test with Library Mode")
+
+
+parent: Path = Path(__file__).parent
+testFiles = glob.glob("test_*.py", root_dir=f"{parent}")
+collect_ignore = []
+if "--with_lib_mode" in sys.argv:
+
+    # ==============================================================================
+    # /!\ Tests are started in Library mode: unselect (ignore) unrelated modules /!\
+    # ==============================================================================
+
+    warnings.warn("Main app is started in library mode")
+
+    configuration.OPTIONAL.MAIN_APP_DIR = "tests/ragger/.test_dependencies/"
+
+    collect_ignore += [f for f in testFiles if "test_clone" not in f]
+
+else:
+
+    # ===========================================================================
+    # /!\ Standards tests without Library mode: unselect (ignore) clone tests /!\
+    # ===========================================================================
+
+    collect_ignore += [f for f in testFiles if "test_clone" in f]
+
+
+#####################
+# CONFIGURATION END #
+#####################
+
 # Pull all features from the base ragger conftest using the overridden configuration
 pytest_plugins = ("ragger.conftest.base_conftest", )

tests/ragger/constants.py

@@ -1,5 +1,3 @@
-from pathlib import Path
 import os
 
-ROOT_SNAPSHOT_PATH = Path(__file__).parent
-ABIS_FOLDER = "%s/abis" % (os.path.dirname(__file__))
+ABIS_FOLDER = f"{os.path.dirname(__file__)}/abis"

tests/ragger/requirements.txt

@@ -1,3 +1,5 @@
 pytest
 ecdsa
-./client/
+web3~=6.0
+ragger[speculos]
+py_ecc