build ci (HemeraProtocol#154)

ideal93 · web-flow · commit 5908a6cf7a17 · 2024-09-23T10:47:48.000+08:00
* Use poetry
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,4 +1,4 @@
-name: Python application test
+name: Python application unit test
 
 on: [pull_request]
 
@@ -16,18 +16,20 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+          pip install --upgrade pip
+          pip install poetry
+          poetry update
+          poetry install -v
 
       - name: Set PYTHONPATH
         run: echo "PYTHONPATH=\$PYTHONPATH:$(pwd)" >> $GITHUB_ENV
 
       - name: Verify PYTHONPATH
         run: echo $PYTHONPATH
 
-      - name: Test with pytest
+      - name: Unit Test with pytest
         env:
-          LINEA_PUBLIC_NODE_RPC_URL: ${{ vars.LINEA_PUBLIC_NODE_RPC_URL }}
+          LINEA_PUBLIC_NODE_RPC_URL: '${{ secrets.LINEA_PUBLIC_NODE_RPC_URL }}'
         run: |
           export PYTHONPATH=$(pwd)
           make test indexer
diff --git a/Dockerfile b/Dockerfile
@@ -1,14 +1,22 @@
-FROM python:3.9-slim
+FROM python:3.9-slim AS builder
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
 
-WORKDIR /app
+RUN pip install poetry && poetry config virtualenvs.in-project true
 
+WORKDIR "/app"
 COPY . .
 
-RUN pip install --no-cache-dir build && \
-    python -m build && \
-    pip install dist/*.whl && \
-    rm dist/*.whl
+RUN poetry install
+RUN poetry build
+
+FROM python:3.9-slim
+
+WORKDIR "/app"
 
-ENV PYTHONPATH=/app:$PYTHONPATH
+COPY --from=builder /app/migrations ./migrations
+COPY --from=builder /app/dist/*.whl .
+RUN pip install *.whl
 
 ENTRYPOINT ["hemera"]
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-VERSION := $(shell cat VERSION)
+VERSION := $(shell poetry version -s)
 BUILD := `git rev-parse --short=7 HEAD`
 SERVICES =
 .PHONY: all build image test
@@ -10,14 +10,14 @@ RESET=\033[0m
 
 image:
 
-	docker build $(IMAGE_FLAGS) --network host -t hemera-protocol:$(VERSION)-$(BUILD) . -q
+	docker build $(IMAGE_FLAGS) --network host -t hemera-protocol:$(VERSION)-$(BUILD) . --no-cache
 	echo "Built image hemera-protocol:$(VERSION)-$(BUILD)"
 
 test:
 	@if [ "$(filter-out $@,$(MAKECMDGOALS))" = "" ]; then \
-		pytest -vv; \
+		poetry run pytest -vv; \
 	else \
-		pytest -vv -m $(filter-out $@,$(MAKECMDGOALS)); \
+		poetry run pytest -vv -m $(filter-out $@,$(MAKECMDGOALS)); \
 	fi
 
 PRE_COMMIT_INSTALLED := $(shell command -v pre-commit > /dev/null 2>&1 && echo yes || echo no)
diff --git a/VERSION b/VERSION
diff --git a/__init__.py b/__init__.py
@@ -1,4 +1,5 @@
 from pathlib import Path
 
-VERSION_FILE = Path(__file__).parent / "VERSION"
-__version__ = VERSION_FILE.read_text().strip()
+import tomli
+
+__version__ = "0.3.0"
diff --git a/cli/__init__.py b/cli/__init__.py
@@ -11,10 +11,11 @@
 
 logging_basic_config()
 
+from importlib import metadata
+
 
 def get_version():
-    version_file = Path(__file__).parent.parent / "VERSION"
-    return version_file.read_text().strip()
+    return metadata.version("hemera")
 
 
 @click.group()
diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml
@@ -9,7 +9,7 @@ x-common-settings: &common-settings
   networks:
     - hemera
   volumes:
-    - ./output:/app/output
+    - hemera-output:/app/output
   depends_on:
     postgresql:
       condition: service_healthy
@@ -19,13 +19,20 @@ services:
     <<: *common-settings
     container_name: indexer
     environment:
+      - AUTO_UPGRADE_DB=true
       - ENTITY_TYPES=EXPLORER_BASE,EXPLORER_TOKEN
       - SYNC_RECORDER=pg:main_recorder
 
   hemera-trace-indexer:
     <<: *common-settings
+    depends_on:
+      postgresql:
+        condition: service_healthy
+      hemera-main-indexer:
+        condition: service_started
     container_name: indexer-trace
     environment:
+      - AUTO_UPGRADE_DB=false
       - ENTITY_TYPES=EXPLORER_TRACE
       - SYNC_RECORDER=pg:trace_recorder
 
@@ -59,7 +66,7 @@ services:
     networks:
       - hemera
     volumes:
-      - ./postgres:/var/lib/postgresql/data
+      - hemera-postgres:/var/lib/postgresql/data
 
   redis:
     image: redis:6
@@ -69,5 +76,9 @@ services:
     networks:
       - hemera
 
+volumes:
+    hemera-postgres:
+    hemera-output:
+
 networks:
   hemera:
diff --git a/docker-compose/hemera-indexer.env b/docker-compose/hemera-indexer.env
@@ -1,7 +1,7 @@
 PROVIDER_URI=https://eth.llamarpc.com
 DEBUG_PROVIDER_URI=https://eth.llamarpc.com
 START_BLOCK=20159954
-POSTGRES_URL=postgresql+psycopg2://user:Ts6YZSrGegXr8PPONtHcSLAHDGd1fjwHBjFZ6WVNpgiOmGf7ghYXl0I1@postgresql:5432/postgres
+POSTGRES_URL=postgresql://user:Ts6YZSrGegXr8PPONtHcSLAHDGd1fjwHBjFZ6WVNpgiOmGf7ghYXl0I1@postgresql:5432/postgres
 OUTPUT=postgres
 
 # This only works when you run this for the very first time. If the container has been created, you will need to remove the volume and recreate the container
diff --git a/hemera_api.py b/hemera_api.py
diff --git a/indexer/domain/token_id_infos.py b/indexer/domain/token_id_infos.py
@@ -28,7 +28,7 @@ def from_token_dict(token_dict: dict):
 class ERC721TokenIdDetail(Domain):
     token_address: str
     token_id: int
-    token_uri: str
+    token_uri: Optional[str]
     block_number: int
     block_timestamp: int
     token_uri_info: Optional[str] = None
diff --git a/indexer/tests/jobs/__init__.py b/indexer/tests/jobs/__init__.py
diff --git a/indexer/tests/jobs/test_token_id_infos_job.py b/indexer/tests/jobs/test_token_id_infos_job.py
@@ -336,99 +336,3 @@ def test_export_token_id_info_job_on_linea_mul():
     )
 
     job_scheduler.clear_data_buff()
-
-
-@pytest.mark.indexer
-@pytest.mark.indexer_exporter
-@pytest.mark.serial
-def test_export_token_id_info_job_on_linea_mul():
-    job_scheduler = JobScheduler(
-        batch_web3_provider=ThreadLocalProxy(
-            lambda: get_provider_from_uri(
-                LINEA_PUBLIC_NODE_RPC_URL,
-                batch=True,
-            )
-        ),
-        batch_web3_debug_provider=ThreadLocalProxy(
-            lambda: get_provider_from_uri(
-                LINEA_PUBLIC_NODE_RPC_URL,
-                batch=True,
-            )
-        ),
-        item_exporters=[ConsoleItemExporter()],
-        batch_size=10,
-        debug_batch_size=1,
-        max_workers=1,
-        config={},
-        required_output_types=[
-            ERC721TokenIdChange,
-            ERC721TokenIdDetail,
-            UpdateERC721TokenIdDetail,
-            ERC1155TokenIdDetail,
-            UpdateERC1155TokenIdDetail,
-        ],
-        multicall=True,
-    )
-
-    job_scheduler.run_jobs(
-        start_block=8494071,
-        end_block=8494071,
-    )
-
-    data_buff = job_scheduler.get_data_buff()
-
-    erc721_token_id_changes = data_buff[ERC721TokenIdChange.type()]
-    assert len(erc721_token_id_changes) == 1
-    assert (
-        ERC721TokenIdChange(
-            token_address="0x6e84390dcc5195414ec91a8c56a5c91021b95704",
-            token_id=110042221770367602542853534930234725702383442308140339620523913150618217206456,
-            token_owner="0xa53cca02f98d590819141aa85c891e2af713c223",
-            block_number=8494071,
-            block_timestamp=1724397109,
-        )
-        in erc721_token_id_changes
-    )
-
-    erc721_token_id_details = data_buff[ERC721TokenIdDetail.type()]
-    assert len(erc721_token_id_details) == 1
-    assert (
-        ERC721TokenIdDetail(
-            token_address="0x6e84390dcc5195414ec91a8c56a5c91021b95704",
-            token_id=110042221770367602542853534930234725702383442308140339620523913150618217206456,
-            token_uri="",
-            block_number=8494071,
-            block_timestamp=1724397109,
-            token_uri_info=None,
-        )
-        in erc721_token_id_details
-    )
-
-    update_erc721_token_id_details = data_buff[UpdateERC721TokenIdDetail.type()]
-    assert len(update_erc721_token_id_details) == 1
-    assert (
-        UpdateERC721TokenIdDetail(
-            token_address="0x6e84390dcc5195414ec91a8c56a5c91021b95704",
-            token_id=110042221770367602542853534930234725702383442308140339620523913150618217206456,
-            token_owner="0xa53cca02f98d590819141aa85c891e2af713c223",
-            block_number=8494071,
-            block_timestamp=1724397109,
-        )
-        in update_erc721_token_id_details
-    )
-
-    erc1155_token_id_details = data_buff[ERC1155TokenIdDetail.type()]
-    assert len(erc1155_token_id_details) == 1
-    assert (
-        ERC1155TokenIdDetail(
-            token_address="0xa53cca02f98d590819141aa85c891e2af713c223",
-            token_id=54780668040604116915679158082040366453838453357839560563054770201457212183923,
-            token_uri="ens-metadata-service.appspot.com/name/0x{id}",
-            block_number=8494071,
-            block_timestamp=1724397109,
-            token_uri_info=None,
-        )
-        in erc1155_token_id_details
-    )
-
-    job_scheduler.clear_data_buff()
diff --git a/indexer/utils/token_fetcher.py b/indexer/utils/token_fetcher.py
@@ -152,7 +152,7 @@ def create_token_detail(self, token_info, value, decode_flag):
 
             if token_info["is_get_token_uri"]:
                 try:
-                    token_uri = decode_string(value) if decode_flag else None
+                    token_uri = decode_string(value) if decode_flag else value
                 except Exception as e:
                     token_uri = None
                     logging.error(f"decode token uri failed, token_info={token_info}, value={value}")
@@ -172,7 +172,7 @@ def create_token_detail(self, token_info, value, decode_flag):
                     return [UpdateERC1155TokenIdDetail(**common_args, token_supply=token_supply)]
         except Exception as e:
             exception_recorder.log(
-                block_number=token_info.block_number,
+                block_number=token_info["block_number"],
                 dataclass=to_snake_case("token_id_info"),
                 message_type="decode_token_id_info_fail",
                 message=str(e),
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
