diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index f2c9c4068..dc51d1421 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -44,6 +44,7 @@ class Settings(BaseSettings): failed_reports_bucket: str = ( "" # for uploading reports that couldn't be sent to fastpath ) + tor_targets: str = "" # filename of json containing Tor bridges and DirAuth endpoints # ooniprobe client configuration collectors: List[Dict[str, str]] = [ diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py index 9f1e13279..b66796a17 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/dependencies.py @@ -1,16 +1,14 @@ -from typing import Annotated, TypeAlias +import io +from functools import lru_cache from pathlib import Path +from typing import Annotated, TypeAlias, Dict, Any -from fastapi import Depends - +import boto3 import geoip2.database - +from clickhouse_driver import Client as Clickhouse +from fastapi import Depends from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker - -from clickhouse_driver import Client as Clickhouse - -import boto3 from mypy_boto3_s3 import S3Client from .common.config import Settings @@ -64,3 +62,23 @@ def get_s3_client() -> S3Client: S3ClientDep = Annotated[S3Client, Depends(get_s3_client)] + + +@lru_cache +def read_file(s3_client : S3ClientDep, bucket: str, file : str) -> str: + """ + Reads the content of `file` within `bucket` into a string + + Useful for reading config files from the s3 bucket + """ + buff = io.BytesIO() + s3_client.download_fileobj(bucket, file, buff) + return buff.getvalue().decode() + + +async def get_tor_targets_from_s3(settings: SettingsDep, s3client: S3ClientDep) -> Dict[str, Any]: + with read_file(s3client, settings.config_bucket, settings.tor_targets) as f: + resp = ujson.load(f) + yield resp + +TorTargetsDep = Annotated[Dict, Depends(get_tor_targets_from_s3)] diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/models.py b/ooniapi/services/ooniprobe/src/ooniprobe/models.py index 1cfc2e104..ddc040d63 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/models.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/models.py @@ -1,10 +1,10 @@ from datetime import datetime -from typing import Dict + +from sqlalchemy import ForeignKey, Sequence, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + from .common.models import UtcDateTime from .common.postgresql import Base -from sqlalchemy import ForeignKey, Sequence, String, Integer -from sqlalchemy.orm import Mapped -from sqlalchemy.orm import mapped_column, relationship class OONIProbeVPNProvider(Base): diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/prio.py b/ooniapi/services/ooniprobe/src/ooniprobe/prio.py index fce2e0167..72c5865cd 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/prio.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/prio.py @@ -21,15 +21,15 @@ ``` """ -from typing import List, Tuple import logging +from typing import List, Tuple + +import sqlalchemy as sa +from clickhouse_driver import Client as Clickhouse from .common.clickhouse_utils import query_click from .common.metrics import timer -from clickhouse_driver import Client as Clickhouse -import sqlalchemy as sa - log = logging.getLogger(__name__) ## Reactive algorithm diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py index 439b0d85b..98cb4a123 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py @@ -1,29 +1,27 @@ -from typing import List, Annotated, Dict, Any import asyncio -from pathlib import Path -import logging -from hashlib import sha512 -from urllib.request import urlopen -from datetime import datetime, timezone import io +import logging import random +from datetime import datetime, timezone +from hashlib import sha512 +from typing import List, Dict, Any -from fastapi import Request, Response, APIRouter, HTTPException, Header, Body import httpx +from fastapi import Request, Response, APIRouter, HTTPException, Header from pydantic import Field from prometheus_client import Counter import zstd +from ..common.metrics import timer +from ..common.routers import BaseModel +from ..common.utils import setnocacheresponse +from ..dependencies import SettingsDep, ASNReaderDep, CCReaderDep, S3ClientDep from ..utils import ( generate_report_id, extract_probe_ipaddr, lookup_probe_cc, lookup_probe_network, ) -from ..dependencies import SettingsDep, ASNReaderDep, CCReaderDep, S3ClientDep -from ..common.routers import BaseModel -from ..common.utils import setnocacheresponse -from ..common.metrics import timer router = APIRouter() diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 6fcca9f15..0929fe58f 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -1,26 +1,31 @@ import logging -from datetime import datetime, timezone, timedelta -import time -from typing import List, Optional, Any, Dict, Tuple, Optional import random +import time +from datetime import datetime, timezone, timedelta +from typing import List, Optional, Any, Dict, Tuple import geoip2 import geoip2.errors -from fastapi import APIRouter, Depends, HTTPException, Response, Request +from fastapi import APIRouter, HTTPException, Response, Request from prometheus_client import Counter, Info, Gauge from pydantic import Field, IPvAnyAddress +from ...common.auth import create_jwt, decode_jwt, jwt +from ...common.routers import BaseModel +from ...common.utils import setnocacheresponse +from ...dependencies import ( + ASNReaderDep, + CCReaderDep, + ClickhouseDep, + SettingsDep, + TorTargetsDep, +) from ...utils import ( generate_report_id, extract_probe_ipaddr, lookup_probe_cc, lookup_probe_network, ) -from ...dependencies import CCReaderDep, ASNReaderDep, ClickhouseDep, SettingsDep -from ...common.routers import BaseModel -from ...common.auth import create_jwt, decode_jwt, jwt -from ...common.config import Settings -from ...common.utils import setnocacheresponse from ...prio import generate_test_list router = APIRouter(prefix="/v1") @@ -636,6 +641,7 @@ class CollectorEntry(BaseModel): front: Optional[str] = Field(default=None, description="Fronted domain") type: Optional[str] = Field(default=None, description="Type of collector") + @router.get("/collectors", tags=["ooniprobe"]) def list_collectors( settings: SettingsDep, @@ -646,3 +652,28 @@ def list_collectors( collector = CollectorEntry(**entry) collectors_response.append(collector) return collectors_response + + +class TorTarget(BaseModel): + address: str + fingerprint: str + name: Optional[str] = '' + protocol: str + params: Optional[Dict[str, List[str]]] = None + + +@router.get("/test-list/tor-targets", tags=["ooniprobe"], response_model=Dict[str, TorTarget]) +def list_tor_targets( + request: Request, + targets: TorTargetsDep, + ) -> Dict[str, TorTarget]: + + token = request.headers.get("Authorization") + if token == None: + # XXX not actually validated + pass + + if targets is not None: + return targets + log.info("tor-targets: failed to receive tor-targets from s3") + raise HTTPException(status_code=401, detail="Invalid tor-targets") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py index 83504d74d..945a7db90 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py @@ -4,23 +4,23 @@ Insert VPN credentials into database. """ -from base64 import b64encode -from os import urandom -from datetime import datetime, timezone import itertools import logging -from typing import List, TypedDict, Tuple import io +from base64 import b64encode +from datetime import datetime, timezone +from os import urandom +from typing import List, TypedDict, Tuple +import httpx +import pem from fastapi import Request from mypy_boto3_s3 import S3Client from sqlalchemy.orm import Session -import pem -import httpx from .common.config import Settings -from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint from .dependencies import CCReaderDep, ASNReaderDep +from ooniprobe.models import OONIProbeVPNProvider, OONIProbeVPNProviderEndpoint RISEUP_CA_URL = "https://api.black.riseup.net/ca.crt" RISEUP_CERT_URL = "https://api.black.riseup.net/3/cert" @@ -147,6 +147,7 @@ def lookup_probe_network(ipaddr: str, asn_reader: ASNReaderDep) -> Tuple[str, st "AS{}".format(resp.autonomous_system_number), resp.autonomous_system_organization or "0", ) + def get_first_ip(headers: str) -> str: """ @@ -158,6 +159,7 @@ def get_first_ip(headers: str) -> str: """ return headers.partition(',')[0] + def read_file(s3_client : S3Client, bucket: str, file : str) -> str: """ Reads the content of `file` within `bucket` into a string @@ -166,4 +168,4 @@ def read_file(s3_client : S3Client, bucket: str, file : str) -> str: """ buff = io.BytesIO() s3_client.download_fileobj(bucket, file, buff) - return buff.getvalue().decode() \ No newline at end of file + return buff.getvalue().decode() diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index a8f53a0eb..7252a44fb 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -5,6 +5,7 @@ import shutil import os import time +from typing import Dict from urllib.request import urlopen from fastapi.testclient import TestClient @@ -12,11 +13,14 @@ from sqlalchemy.orm import sessionmaker from clickhouse_driver import Client as ClickhouseClient +import ujson + from ooniprobe.common.config import Settings from ooniprobe.common.dependencies import get_settings -from ooniprobe.dependencies import get_s3_client +from ooniprobe.dependencies import get_s3_client, get_tor_targets_from_s3 from ooniprobe.main import app from ooniprobe.download_geoip import try_update +from ooniprobe.routers.v1.probe_services import TorTarget def make_override_get_settings(**kw): @@ -101,6 +105,7 @@ def geoip_db_dir(fixture_path): def client(clickhouse_server, test_settings, geoip_db_dir): app.dependency_overrides[get_settings] = test_settings app.dependency_overrides[get_s3_client] = get_s3_client_mock + app.dependency_overrides[get_tor_targets_from_s3] = get_tor_targets_from_s3_mock # lifespan won't run so do this here to have the DB try_update(geoip_db_dir) client = TestClient(app) @@ -116,7 +121,8 @@ def test_settings(alembic_migration, geoip_db_dir, clickhouse_server, fastpath_s clickhouse_url=clickhouse_server, geoip_db_dir=geoip_db_dir, collector_id="1", - fastpath_url=fastpath_server + fastpath_url=fastpath_server, + tor_targets="./tests/fixtures/data/tor-targets.json" ) @@ -162,6 +168,10 @@ def upload_fileobj(self, Fileobj, Bucket: str, Key: str): def get_s3_client_mock() -> S3ClientMock: return S3ClientMock() +def get_tor_targets_from_s3_mock() -> Dict[str, TorTarget]: + with open("./tests/fixtures/data/tor-targets.json", "r") as f: + yield ujson.load(f) + @pytest.fixture(scope="session") def fastpath_server(docker_ip, docker_services): port = docker_services.port_for("fakepath", 80) diff --git a/ooniapi/services/ooniprobe/tests/fixtures/data/tor-targets.json b/ooniapi/services/ooniprobe/tests/fixtures/data/tor-targets.json new file mode 100644 index 000000000..82edcd142 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/fixtures/data/tor-targets.json @@ -0,0 +1 @@ +{"128.31.0.39:9201":{"address":"128.31.0.39:9201","fingerprint":"1A25C6358DB91342AA51720A5038B72742732498","name":"moria1","protocol":"or_port_dirauth"},"128.31.0.39:9231":{"address":"128.31.0.39:9231","fingerprint":"1A25C6358DB91342AA51720A5038B72742732498","name":"moria1","protocol":"dir_port"},"131.188.40.189:443":{"address":"131.188.40.189:443","fingerprint":"F2044413DAC2E02E3D6BCF4735A19BCA1DE97281","name":"gabelmoo","protocol":"or_port_dirauth"},"131.188.40.189:80":{"address":"131.188.40.189:80","fingerprint":"F2044413DAC2E02E3D6BCF4735A19BCA1DE97281","name":"gabelmoo","protocol":"dir_port"},"171.25.193.9:443":{"address":"171.25.193.9:443","fingerprint":"BD6A829255CB08E66FBE7D3748363586E46B3810","name":"maatuska","protocol":"dir_port"},"171.25.193.9:80":{"address":"171.25.193.9:80","fingerprint":"BD6A829255CB08E66FBE7D3748363586E46B3810","name":"maatuska","protocol":"or_port_dirauth"},"193.23.244.244:443":{"address":"193.23.244.244:443","fingerprint":"7BE683E65D48141321C5ED92F075C55364AC7123","name":"dannenberg","protocol":"or_port_dirauth"},"193.23.244.244:80":{"address":"193.23.244.244:80","fingerprint":"7BE683E65D48141321C5ED92F075C55364AC7123","name":"dannenberg","protocol":"dir_port"},"199.58.81.140:443":{"address":"199.58.81.140:443","fingerprint":"74A910646BCEEFBCD2E874FC1DC997430F968145","name":"longclaw","protocol":"or_port_dirauth"},"199.58.81.140:80":{"address":"199.58.81.140:80","fingerprint":"74A910646BCEEFBCD2E874FC1DC997430F968145","name":"longclaw","protocol":"dir_port"},"204.13.164.118:443":{"address":"204.13.164.118:443","fingerprint":"24E2F139121D4394C54B5BCC368B3B411857C413","name":"bastet","protocol":"or_port_dirauth"},"204.13.164.118:80":{"address":"204.13.164.118:80","fingerprint":"24E2F139121D4394C54B5BCC368B3B411857C413","name":"bastet","protocol":"dir_port"},"216.218.219.41:443":{"address":"216.218.219.41:443","fingerprint":"E3E42D35F801C9D5AB23584E0025D56FE2B33396","name":"Faravahar","protocol":"or_port_dirauth"},"216.218.219.41:80":{"address":"216.218.219.41:80","fingerprint":"E3E42D35F801C9D5AB23584E0025D56FE2B33396","name":"Faravahar","protocol":"dir_port"},"217.196.147.77:443":{"address":"217.196.147.77:443","fingerprint":"FAA4BCA4A6AC0FB4CA2F8AD5A11D9E122BA894F6","name":"tor26","protocol":"or_port_dirauth"},"217.196.147.77:80":{"address":"217.196.147.77:80","fingerprint":"FAA4BCA4A6AC0FB4CA2F8AD5A11D9E122BA894F6","name":"tor26","protocol":"dir_port"},"2d7292b5163fb7de5b24cd04032c93a2d4c454431de3a00b5a6d4a3309529e49":{"address":"193.11.166.194:27020","fingerprint":"86AC7B8D430DAC4117E9F42C9EAED18133863AAF","params":{"cert":["0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg"],"iat-mode":["0"]},"protocol":"obfs4"},"3fa772a44e07856b4c70e958b2f6dc8a29450a823509d5dbbf8b884e7fb5bb9d":{"address":"192.95.36.142:443","fingerprint":"CDF2E852BF539B82BD10E27E9115A31734E378C2","params":{"cert":["qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ"],"iat-mode":["1"]},"protocol":"obfs4"},"45.66.35.11:443":{"address":"45.66.35.11:443","fingerprint":"7EA6EAD6FD83083C538F44038BBFA077587DD755","name":"dizum","protocol":"or_port_dirauth"},"45.66.35.11:80":{"address":"45.66.35.11:80","fingerprint":"7EA6EAD6FD83083C538F44038BBFA077587DD755","name":"dizum","protocol":"dir_port"},"49116bf72d336bb8724fd3a06a5afa7bbd4e7baef35fbcdb9a98d13e702270ad":{"address":"146.57.248.225:22","fingerprint":"10A6CD36A537FCE513A322361547444B393989F0","params":{"cert":["K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw"],"iat-mode":["0"]},"protocol":"obfs4"},"4a330634c5d678887f0f7c299490af43a6ac9fa944a6cc2140ab264c9ec124a0":{"address":"209.148.46.65:443","fingerprint":"74FAD13168806246602538555B5521A0383A1875","params":{"cert":["ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw"],"iat-mode":["0"]},"protocol":"obfs4"},"548eebff71da6128321c3bc1c3ec12b5bfff277ef5cde32709a33e207b57f3e2":{"address":"37.218.245.14:38224","fingerprint":"D9A82D2F9C2F65A18407B1D2B764F130847F8B5D","params":{"cert":["bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg"],"iat-mode":["0"]},"protocol":"obfs4"},"5aeb9e43b43fc8a809b8d25aae968395a5ceea0e677caaf56e1c0a2ba002f5b5":{"address":"193.11.166.194:27015","fingerprint":"2D82C2E354D531A68469ADF7F878FA6060C6BACA","params":{"cert":["4TLQPJrTSaDffMK7Nbao6LC7G9OW/NHkUwIdjLSS3KYf0Nv4/nQiiI8dY2TcsQx01NniOg"],"iat-mode":["0"]},"protocol":"obfs4"},"66.111.2.131:9001":{"address":"66.111.2.131:9001","fingerprint":"BA44A889E64B93FAA2B114E02C2A279A8555C533","name":"Serge","protocol":"or_port_dirauth"},"66.111.2.131:9030":{"address":"66.111.2.131:9030","fingerprint":"BA44A889E64B93FAA2B114E02C2A279A8555C533","name":"Serge","protocol":"dir_port"},"662218447d396b9d4f01b585457d267735601fedbeb9a19b86b942f238fe4e7b":{"address":"51.222.13.177:80","fingerprint":"5EDAC3B810E12B01F6FD8050D2FD3E277B289A08","params":{"cert":["2uplIpLQ0q9+0qMFrK5pkaYRDOe460LL9WHBvatgkuRr/SL31wBOEupaMMJ6koRE6Ld0ew"],"iat-mode":["0"]},"protocol":"obfs4"},"75fe96d641a078fee06529af376d7f8c92757596e48558d5d02baa1e10321d10":{"address":"45.145.95.6:27015","fingerprint":"C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C","params":{"cert":["TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw"],"iat-mode":["0"]},"protocol":"obfs4"},"99e9adc8bba0d60982dbc655b5e8735d88ad788905c3713a39eff3224b617eeb":{"address":"38.229.1.78:80","fingerprint":"C8CBDB2464FC9804A69531437BCF2BE31FDD2EE4","params":{"cert":["Hmyfd2ev46gGY7NoVxA9ngrPF2zCZtzskRTzoWXbxNkzeVnGFPWmrTtILRyqCTjHR+s9dg"],"iat-mode":["1"]},"protocol":"obfs4"},"9d735c6e70512123ab2c2fe966446b2345b352c512e9fb359f4b1673236e4d4a":{"address":"38.229.33.83:80","fingerprint":"0BAC39417268B96B9F514E7F63FA6FBA1A788955","params":{"cert":["VwEFpk9F/UN9JED7XpG1XOjm/O8ZCXK80oPecgWnNDZDv5pdkhq1OpbAH0wNqOT6H6BmRQ"],"iat-mode":["1"]},"protocol":"obfs4"},"b7c0e3f183ad85a6686ec68344765cec57906b215e7b82a98a9ca013cb980efa":{"address":"193.11.166.194:27025","fingerprint":"1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF","params":{"cert":["ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA"],"iat-mode":["0"]},"protocol":"obfs4"},"b8de51da541ced804840b1d8fd24d5ff1cfdf07eae673dae38c2bc2cce594ddd":{"address":"85.31.186.26:443","fingerprint":"91A6354697E6B02A386312F68D82CF86824D3606","params":{"cert":["PBwr+S8JTVZo6MPdHnkTwXJPILWADLqfMGoVvhZClMq/Urndyd42BwX9YFJHZnBB3H0XCw"],"iat-mode":["0"]},"protocol":"obfs4"},"d2d6e34abeda851f7cd37138ffafcce992b2ccdb0f263eb90ab75d7adbd5eeba":{"address":"85.31.186.98:443","fingerprint":"011F2599C0E9B27EE74B353155E244813763C3E5","params":{"cert":["ayq0XzCwhpdysn5o0EyDUbmSOx3X/oTEbzDMvczHOdBJKlvIdHHLJGkZARtT4dcBFArPPg"],"iat-mode":["0"]},"protocol":"obfs4"},"f855ba38d517d8589c16e1333ac23c6e516532cf036ab6f47b15030b40a3b6a6":{"address":"[2a0c:4d80:42:702::1]:27015","fingerprint":"C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C","params":{"cert":["TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw"],"iat-mode":["0"]},"protocol":"obfs4"}} diff --git a/ooniapi/services/ooniprobe/tests/integ/test_tor_targets.py b/ooniapi/services/ooniprobe/tests/integ/test_tor_targets.py new file mode 100644 index 000000000..0c725fef7 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/integ/test_tor_targets.py @@ -0,0 +1,6 @@ +def test_tor_targets(client): + resp = client.get("/api/v1/test-list/tor-targets").json() + for i, target in resp.items(): + assert i is not None + for k in ["address", "fingerprint", "name", "protocol"]: + assert k in target