add test-list/tor-targets to ooniapi probe services

ooniapi/common/src/common/config.py

@@ -44,6 +44,7 @@ class Settings(BaseSettings):
     failed_reports_bucket: str = (
         ""  # for uploading reports that couldn't be sent to fastpath
     )
+    tor_targets: str = "" # filename of json containing Tor bridges and DirAuth endpoints
 
     # ooniprobe client configuration
     collectors: List[Dict[str, str]] = [

ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py

@@ -1,16 +1,14 @@
-from typing import Annotated, TypeAlias
+import io
+from functools import lru_cache
 from pathlib import Path
+from typing import Annotated, TypeAlias, Dict, Any
 
-from fastapi import Depends
-
+import boto3
 import geoip2.database
-
+from clickhouse_driver import Client as Clickhouse
+from fastapi import Depends
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
-
-from clickhouse_driver import Client as Clickhouse
-
-import boto3
 from mypy_boto3_s3 import S3Client
 
 from .common.config import Settings
@@ -64,3 +62,23 @@ def get_s3_client() -> S3Client:
 
 
 S3ClientDep = Annotated[S3Client, Depends(get_s3_client)]
+
+
+@lru_cache
+def read_file(s3_client : S3ClientDep, bucket: str, file : str) -> str:
+    """
+    Reads the content of `file` within `bucket` into a  string
+
+    Useful for reading config files from the s3 bucket
+    """
+    buff = io.BytesIO()
+    s3_client.download_fileobj(bucket, file, buff)
+    return buff.getvalue().decode()
+
+
+async def get_tor_targets_from_s3(settings: SettingsDep, s3client: S3ClientDep) -> Dict[str, Any]:
+    with read_file(s3client, settings.config_bucket, settings.tor_targets) as f:
+        resp = ujson.load(f)
+    yield resp
+
+TorTargetsDep = Annotated[Dict, Depends(get_tor_targets_from_s3)]

ooniapi/services/ooniprobe/src/ooniprobe/models.py

@@ -1,10 +1,10 @@
 from datetime import datetime
-from typing import Dict
+
+from sqlalchemy import ForeignKey, Sequence, String
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
 from .common.models import UtcDateTime
 from .common.postgresql import Base
-from sqlalchemy import ForeignKey, Sequence, String, Integer
-from sqlalchemy.orm import Mapped
-from sqlalchemy.orm import mapped_column, relationship
 
 
 class OONIProbeVPNProvider(Base):

ooniapi/services/ooniprobe/src/ooniprobe/prio.py

@@ -21,15 +21,15 @@
 ```
 """
 
-from typing import List, Tuple
 import logging
+from typing import List, Tuple
+
+import sqlalchemy as sa
+from clickhouse_driver import Client as Clickhouse
 
 from .common.clickhouse_utils import query_click
 from .common.metrics import timer
 
-from clickhouse_driver import Client as Clickhouse
-import sqlalchemy as sa
-
 log = logging.getLogger(__name__)
 
 ## Reactive algorithm

ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py

@@ -1,29 +1,27 @@
-from typing import List, Annotated, Dict, Any
 import asyncio
-from pathlib import Path
-import logging
-from hashlib import sha512
-from urllib.request import urlopen
-from datetime import datetime, timezone
 import io
+import logging
 import random
+from datetime import datetime, timezone
+from hashlib import sha512
+from typing import List, Dict, Any
 
-from fastapi import Request, Response, APIRouter, HTTPException, Header, Body
 import httpx
+from fastapi import Request, Response, APIRouter, HTTPException, Header
 from pydantic import Field
 from prometheus_client import Counter
 import zstd
 
+from ..common.metrics import timer
+from ..common.routers import BaseModel
+from ..common.utils import setnocacheresponse
+from ..dependencies import SettingsDep, ASNReaderDep, CCReaderDep, S3ClientDep
 from ..utils import (
     generate_report_id,
     extract_probe_ipaddr,
     lookup_probe_cc,
     lookup_probe_network,
 )
-from ..dependencies import SettingsDep, ASNReaderDep, CCReaderDep, S3ClientDep
-from ..common.routers import BaseModel
-from ..common.utils import setnocacheresponse
-from ..common.metrics import timer
 
 router = APIRouter()
 

ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py

@@ -1,26 +1,31 @@
 import logging
-from datetime import datetime, timezone, timedelta
-import time
-from typing import List, Optional, Any, Dict, Tuple, Optional
 import random
+import time
+from datetime import datetime, timezone, timedelta
+from typing import List, Optional, Any, Dict, Tuple
 
 import geoip2
 import geoip2.errors
-from fastapi import APIRouter, Depends, HTTPException, Response, Request
+from fastapi import APIRouter, HTTPException, Response, Request
 from prometheus_client import Counter, Info, Gauge
 from pydantic import Field, IPvAnyAddress
 
+from ...common.auth import create_jwt, decode_jwt, jwt
+from ...common.routers import BaseModel
+from ...common.utils import setnocacheresponse
+from ...dependencies import (
+    ASNReaderDep,
+    CCReaderDep,
+    ClickhouseDep,
+    SettingsDep,
+    TorTargetsDep,
+)
 from ...utils import (
     generate_report_id,
     extract_probe_ipaddr,
     lookup_probe_cc,
     lookup_probe_network,
 )
-from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep
-from ...common.routers import BaseModel
-from ...common.auth import create_jwt, decode_jwt, jwt
-from ...common.config import Settings
-from ...common.utils import setnocacheresponse
 from ...prio import generate_test_list
 
 router = APIRouter(prefix="/v1")
@@ -636,6 +641,7 @@ class CollectorEntry(BaseModel):
     front: Optional[str] = Field(default=None, description="Fronted domain")
     type: Optional[str] = Field(default=None, description="Type of collector")
 
+
 @router.get("/collectors", tags=["ooniprobe"])
 def list_collectors(
     settings: SettingsDep,
@@ -646,3 +652,28 @@ def list_collectors(
         collector = CollectorEntry(**entry)
         collectors_response.append(collector)
     return collectors_response
+
+
+class TorTarget(BaseModel):
+    address: str
+    fingerprint: str
+    name: Optional[str] = ''
+    protocol: str
+    params: Optional[Dict[str, List[str]]] = None
+
+
+@router.get("/test-list/tor-targets", tags=["ooniprobe"], response_model=Dict[str, TorTarget])
+def list_tor_targets(
+    request: Request,
+    targets: TorTargetsDep,
+    ) -> Dict[str, TorTarget]:
+
+    token = request.headers.get("Authorization")
+    if token == None:
+        # XXX not actually validated
+        pass
+
+    if targets is not None:
+        return targets
+    log.info("tor-targets: failed to receive tor-targets from s3")
+    raise HTTPException(status_code=401, detail="Invalid tor-targets")

ooniapi/services/ooniprobe/src/ooniprobe/utils.py

@@ -4,23 +4,23 @@
 Insert VPN credentials into database.
 """
 
-from base64 import b64encode
-from os import urandom
-from datetime import datetime, timezone
 import itertools
 import logging
-from typing import List, TypedDict, Tuple
 import io
+from base64 import b64encode
+from datetime import datetime, timezone
+from os import urandom
+from typing import List, TypedDict, Tuple
 
+import httpx
+import pem
 from fastapi import Request
 from mypy_boto3_s3 import S3Client
 from sqlalchemy.orm import Session
-import pem
-import httpx
 
 from .common.config import Settings
-from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint
 from .dependencies import CCReaderDep, ASNReaderDep
+from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint
 
 RISEUP_CA_URL = "https://api.black.riseup.net/ca.crt"
 RISEUP_CERT_URL = "https://api.black.riseup.net/3/cert"
@@ -147,6 +147,7 @@ def lookup_probe_network(ipaddr: str, asn_reader: ASNReaderDep) -> Tuple[str, st
         "AS{}".format(resp.autonomous_system_number),
         resp.autonomous_system_organization or "0",
     )
+
 
 def get_first_ip(headers: str) -> str:
     """
@@ -158,6 +159,7 @@ def get_first_ip(headers: str) -> str:
     """
     return headers.partition(',')[0]
 
+
 def read_file(s3_client : S3Client, bucket: str, file : str) -> str:
     """
     Reads the content of `file` within `bucket` into a  string
@@ -166,4 +168,4 @@ def read_file(s3_client : S3Client, bucket: str, file : str) -> str:
     """
     buff = io.BytesIO()
     s3_client.download_fileobj(bucket, file, buff)
-    return buff.getvalue().decode()
+    return buff.getvalue().decode()

ooniapi/services/ooniprobe/tests/conftest.py

@@ -5,18 +5,22 @@
 import shutil
 import os
 import time
+from typing import Dict
 from urllib.request import urlopen
 
 from fastapi.testclient import TestClient
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from clickhouse_driver import Client as ClickhouseClient
 
+import ujson
+
 from ooniprobe.common.config import Settings
 from ooniprobe.common.dependencies import get_settings
-from ooniprobe.dependencies import get_s3_client
+from ooniprobe.dependencies import get_s3_client, get_tor_targets_from_s3
 from ooniprobe.main import app
 from ooniprobe.download_geoip import try_update
+from ooniprobe.routers.v1.probe_services import TorTarget
 
 
 def make_override_get_settings(**kw):
@@ -101,6 +105,7 @@ def geoip_db_dir(fixture_path):
 def client(clickhouse_server, test_settings, geoip_db_dir):
     app.dependency_overrides[get_settings] = test_settings
     app.dependency_overrides[get_s3_client] = get_s3_client_mock
+    app.dependency_overrides[get_tor_targets_from_s3] = get_tor_targets_from_s3_mock
     # lifespan won't run so do this here to have the DB
     try_update(geoip_db_dir)
     client = TestClient(app)
@@ -116,7 +121,8 @@ def test_settings(alembic_migration, geoip_db_dir, clickhouse_server, fastpath_s
         clickhouse_url=clickhouse_server,
         geoip_db_dir=geoip_db_dir,
         collector_id="1",
-        fastpath_url=fastpath_server
+        fastpath_url=fastpath_server,
+        tor_targets="./tests/fixtures/data/tor-targets.json"
     )
 
 
@@ -162,6 +168,10 @@ def upload_fileobj(self, Fileobj, Bucket: str, Key: str):
 def get_s3_client_mock() -> S3ClientMock:
     return S3ClientMock()
 
+def get_tor_targets_from_s3_mock() -> Dict[str, TorTarget]:
+    with open("./tests/fixtures/data/tor-targets.json", "r") as f:
+        yield ujson.load(f)
+
 @pytest.fixture(scope="session")
 def fastpath_server(docker_ip, docker_services):
     port = docker_services.port_for("fakepath", 80)

ooniapi/services/ooniprobe/tests/fixtures/data/tor-targets.json

@@ -0,0 +1 @@
+{"128.31.0.39:9201":{"address":"128.31.0.39:9201","fingerprint":"1A25C6358DB91342AA51720A5038B72742732498","name":"moria1","protocol":"or_port_dirauth"},"128.31.0.39:9231":{"address":"128.31.0.39:9231","fingerprint":"1A25C6358DB91342AA51720A5038B72742732498","name":"moria1","protocol":"dir_port"},"131.188.40.189:443":{"address":"131.188.40.189:443","fingerprint":"F2044413DAC2E02E3D6BCF4735A19BCA1DE97281","name":"gabelmoo","protocol":"or_port_dirauth"},"131.188.40.189:80":{"address":"131.188.40.189:80","fingerprint":"F2044413DAC2E02E3D6BCF4735A19BCA1DE97281","name":"gabelmoo","protocol":"dir_port"},"171.25.193.9:443":{"address":"171.25.193.9:443","fingerprint":"BD6A829255CB08E66FBE7D3748363586E46B3810","name":"maatuska","protocol":"dir_port"},"171.25.193.9:80":{"address":"171.25.193.9:80","fingerprint":"BD6A829255CB08E66FBE7D3748363586E46B3810","name":"maatuska","protocol":"or_port_dirauth"},"193.23.244.244:443":{"address":"193.23.244.244:443","fingerprint":"7BE683E65D48141321C5ED92F075C55364AC7123","name":"dannenberg","protocol":"or_port_dirauth"},"193.23.244.244:80":{"address":"193.23.244.244:80","fingerprint":"7BE683E65D48141321C5ED92F075C55364AC7123","name":"dannenberg","protocol":"dir_port"},"199.58.81.140:443":{"address":"199.58.81.140:443","fingerprint":"74A910646BCEEFBCD2E874FC1DC997430F968145","name":"longclaw","protocol":"or_port_dirauth"},"199.58.81.140:80":{"address":"199.58.81.140:80","fingerprint":"74A910646BCEEFBCD2E874FC1DC997430F968145","name":"longclaw","protocol":"dir_port"},"204.13.164.118:443":{"address":"204.13.164.118:443","fingerprint":"24E2F139121D4394C54B5BCC368B3B411857C413","name":"bastet","protocol":"or_port_dirauth"},"204.13.164.118:80":{"address":"204.13.164.118:80","fingerprint":"24E2F139121D4394C54B5BCC368B3B411857C413","name":"bastet","protocol":"dir_port"},"216.218.219.41:443":{"address":"216.218.219.41:443","fingerprint":"E3E42D35F801C9D5AB23584E0025D56FE2B33396","name":"Faravahar","protocol":"or_port_dirauth"},"216.218.219.41:80":{"address":"216.218.219.41:80","fingerprint":"E3E42D35F801C9D5AB23584E0025D56FE2B33396","name":"Faravahar","protocol":"dir_port"},"217.196.147.77:443":{"address":"217.196.147.77:443","fingerprint":"FAA4BCA4A6AC0FB4CA2F8AD5A11D9E122BA894F6","name":"tor26","protocol":"or_port_dirauth"},"217.196.147.77:80":{"address":"217.196.147.77:80","fingerprint":"FAA4BCA4A6AC0FB4CA2F8AD5A11D9E122BA894F6","name":"tor26","protocol":"dir_port"},"2d7292b5163fb7de5b24cd04032c93a2d4c454431de3a00b5a6d4a3309529e49":{"address":"193.11.166.194:27020","fingerprint":"86AC7B8D430DAC4117E9F42C9EAED18133863AAF","params":{"cert":["0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg"],"iat-mode":["0"]},"protocol":"obfs4"},"3fa772a44e07856b4c70e958b2f6dc8a29450a823509d5dbbf8b884e7fb5bb9d":{"address":"192.95.36.142:443","fingerprint":"CDF2E852BF539B82BD10E27E9115A31734E378C2","params":{"cert":["qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ"],"iat-mode":["1"]},"protocol":"obfs4"},"45.66.35.11:443":{"address":"45.66.35.11:443","fingerprint":"7EA6EAD6FD83083C538F44038BBFA077587DD755","name":"dizum","protocol":"or_port_dirauth"},"45.66.35.11:80":{"address":"45.66.35.11:80","fingerprint":"7EA6EAD6FD83083C538F44038BBFA077587DD755","name":"dizum","protocol":"dir_port"},"49116bf72d336bb8724fd3a06a5afa7bbd4e7baef35fbcdb9a98d13e702270ad":{"address":"146.57.248.225:22","fingerprint":"10A6CD36A537FCE513A322361547444B393989F0","params":{"cert":["K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw"],"iat-mode":["0"]},"protocol":"obfs4"},"4a330634c5d678887f0f7c299490af43a6ac9fa944a6cc2140ab264c9ec124a0":{"address":"209.148.46.65:443","fingerprint":"74FAD13168806246602538555B5521A0383A1875","params":{"cert":["ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw"],"iat-mode":["0"]},"protocol":"obfs4"},"548eebff71da6128321c3bc1c3ec12b5bfff277ef5cde32709a33e207b57f3e2":{"address":"37.218.245.14:38224","fingerprint":"D9A82D2F9C2F65A18407B1D2B764F130847F8B5D","params":{"cert":["bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg"],"iat-mode":["0"]},"protocol":"obfs4"},"5aeb9e43b43fc8a809b8d25aae968395a5ceea0e677caaf56e1c0a2ba002f5b5":{"address":"193.11.166.194:27015","fingerprint":"2D82C2E354D531A68469ADF7F878FA6060C6BACA","params":{"cert":["4TLQPJrTSaDffMK7Nbao6LC7G9OW/NHkUwIdjLSS3KYf0Nv4/nQiiI8dY2TcsQx01NniOg"],"iat-mode":["0"]},"protocol":"obfs4"},"66.111.2.131:9001":{"address":"66.111.2.131:9001","fingerprint":"BA44A889E64B93FAA2B114E02C2A279A8555C533","name":"Serge","protocol":"or_port_dirauth"},"66.111.2.131:9030":{"address":"66.111.2.131:9030","fingerprint":"BA44A889E64B93FAA2B114E02C2A279A8555C533","name":"Serge","protocol":"dir_port"},"662218447d396b9d4f01b585457d267735601fedbeb9a19b86b942f238fe4e7b":{"address":"51.222.13.177:80","fingerprint":"5EDAC3B810E12B01F6FD8050D2FD3E277B289A08","params":{"cert":["2uplIpLQ0q9+0qMFrK5pkaYRDOe460LL9WHBvatgkuRr/SL31wBOEupaMMJ6koRE6Ld0ew"],"iat-mode":["0"]},"protocol":"obfs4"},"75fe96d641a078fee06529af376d7f8c92757596e48558d5d02baa1e10321d10":{"address":"45.145.95.6:27015","fingerprint":"C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C","params":{"cert":["TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw"],"iat-mode":["0"]},"protocol":"obfs4"},"99e9adc8bba0d60982dbc655b5e8735d88ad788905c3713a39eff3224b617eeb":{"address":"38.229.1.78:80","fingerprint":"C8CBDB2464FC9804A69531437BCF2BE31FDD2EE4","params":{"cert":["Hmyfd2ev46gGY7NoVxA9ngrPF2zCZtzskRTzoWXbxNkzeVnGFPWmrTtILRyqCTjHR+s9dg"],"iat-mode":["1"]},"protocol":"obfs4"},"9d735c6e70512123ab2c2fe966446b2345b352c512e9fb359f4b1673236e4d4a":{"address":"38.229.33.83:80","fingerprint":"0BAC39417268B96B9F514E7F63FA6FBA1A788955","params":{"cert":["VwEFpk9F/UN9JED7XpG1XOjm/O8ZCXK80oPecgWnNDZDv5pdkhq1OpbAH0wNqOT6H6BmRQ"],"iat-mode":["1"]},"protocol":"obfs4"},"b7c0e3f183ad85a6686ec68344765cec57906b215e7b82a98a9ca013cb980efa":{"address":"193.11.166.194:27025","fingerprint":"1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF","params":{"cert":["ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA"],"iat-mode":["0"]},"protocol":"obfs4"},"b8de51da541ced804840b1d8fd24d5ff1cfdf07eae673dae38c2bc2cce594ddd":{"address":"85.31.186.26:443","fingerprint":"91A6354697E6B02A386312F68D82CF86824D3606","params":{"cert":["PBwr+S8JTVZo6MPdHnkTwXJPILWADLqfMGoVvhZClMq/Urndyd42BwX9YFJHZnBB3H0XCw"],"iat-mode":["0"]},"protocol":"obfs4"},"d2d6e34abeda851f7cd37138ffafcce992b2ccdb0f263eb90ab75d7adbd5eeba":{"address":"85.31.186.98:443","fingerprint":"011F2599C0E9B27EE74B353155E244813763C3E5","params":{"cert":["ayq0XzCwhpdysn5o0EyDUbmSOx3X/oTEbzDMvczHOdBJKlvIdHHLJGkZARtT4dcBFArPPg"],"iat-mode":["0"]},"protocol":"obfs4"},"f855ba38d517d8589c16e1333ac23c6e516532cf036ab6f47b15030b40a3b6a6":{"address":"[2a0c:4d80:42:702::1]:27015","fingerprint":"C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C","params":{"cert":["TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw"],"iat-mode":["0"]},"protocol":"obfs4"}}

ooniapi/services/ooniprobe/tests/integ/test_tor_targets.py

@@ -0,0 +1,6 @@
+def test_tor_targets(client):
+    resp = client.get("/api/v1/test-list/tor-targets").json()
+    for i, target in resp.items():
+        assert i is not None
+        for k in ["address", "fingerprint", "name", "protocol"]:
+            assert k in target