diff --git a/.gitignore b/.gitignore index de18f02c7..9a3c75751 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,7 @@ scripts/*client.json # Vs code settings .vscode/ + +# My Entries +notes.txt +Temp/ diff --git a/configurations/default/env.yml.tmp b/configurations/default/env.yml.tmp index efcad4a54..8b3b35da4 100644 --- a/configurations/default/env.yml.tmp +++ b/configurations/default/env.yml.tmp @@ -2,14 +2,13 @@ AUTH0_CLIENT_ID: your-auth0-client-id AUTH0_CONNECTION_NAME: your-auth0-connection-name AUTH0_DOMAIN: your-auth0-domain BUGSNAG_KEY: optional-bugsnag-key -MAP_BASE_URL: optional-map-tile-url +MAP_BASE_URL: http://tile.openstreetmap.org/{z}/{x}/{y}.png # Uncomment it if maps are gray MAPBOX_ACCESS_TOKEN: your-mapbox-access-token MAPBOX_MAP_ID: mapbox/outdoors-v11 MAPBOX_ATTRIBUTION: © Mapbox © OpenStreetMap Improve this map -# MAP_BASE_URL: http://tile.openstreetmap.org/{z}/{x}/{y}.png # Uncomment it if maps are gray SLACK_CHANNEL: optional-slack-channel SLACK_WEBHOOK: optional-slack-webhook -GRAPH_HOPPER_KEY: your-graph-hopper-key +# GRAPH_HOPPER_KEY: your-graph-hopper-key # Optional override to use a custom service instead of the graphhopper.com hosted service. # GRAPH_HOPPER_URL: http://localhost:8989/ # Optional overrides to use custom service or different api key for certain bounding boxes. @@ -25,3 +24,6 @@ GRAPH_HOPPER_KEY: your-graph-hopper-key GOOGLE_ANALYTICS_TRACKING_ID: optional-ga-key # GRAPH_HOPPER_POINT_LIMIT: 10 # Defaults to 30 DISABLE_AUTH: true + +MAPTYPE: OSM +VALHALLA_URL: "https://valhalla1.openstreetmap.de/route" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9055475a6..2f43e4367 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" x-common-variables: &common-variables - BUGSNAG_KEY=${BUGSNAG_KEY} @@ -42,6 +41,7 @@ services: target: /config ports: - "4000:4000" + datatools-ui: build: context: ../ @@ -51,6 +51,7 @@ services: environment: *common-variables ports: - "9966:9966" + mongo: image: mongo restart: always diff --git a/docker/ui/Dockerfile b/docker/ui/Dockerfile index a3f1bc811..baeecfd19 100644 --- a/docker/ui/Dockerfile +++ b/docker/ui/Dockerfile @@ -6,11 +6,12 @@ ARG BUGSNAG_KEY RUN cd /datatools-build COPY package.json yarn.lock patches /datatools-build/ RUN yarn -COPY . /datatools-build/ +COPY . /datatools-build/ +COPY ./lib/mastarm_css/css-transform.js /datatools-build/node_modules/mastarm/lib/css-transform.js COPY configurations/default /datatools-config/ # Copy the tmp file to the env.yml if no env.yml is present RUN cp -R -u -p /datatools-config/env.yml.tmp /datatools-config/env.yml -CMD yarn run mastarm build --env dev --serve --proxy http://datatools-server:4000/api # \ No newline at end of file +CMD yarn run mastarm build --env dev --serve --proxy http://datatools-server:4000/api # diff --git a/lib/mastarm_css/css-transform.js b/lib/mastarm_css/css-transform.js new file mode 100644 index 000000000..d0df644f5 --- /dev/null +++ b/lib/mastarm_css/css-transform.js @@ -0,0 +1,124 @@ +const fs = require('fs') +const path = require('path') + +const cssnano = require('cssnano') +const chokidar = require('chokidar') +const mimeType = require('mime') +const mkdirp = require('mkdirp') +const postcss = require('postcss') +const postcssImport = require('postcss-import') +const postcssPresetEnv = require('postcss-preset-env') +const postcssReporter = require('postcss-reporter') +const postcssSafeParser = require('postcss-safe-parser') + +const browsers = require('./constants').BROWSER_SUPPORT +const logger = require('./logger') + +module.exports = function ({ config, entry, minify, outfile, watch }) { + const configImport = config.stylePath + ? `@import url(${config.stylePath});` + : '' + let watcher + const plugins = [ + postcssImport({ + onImport: sources => { + if (watch) { + sources.forEach(source => watcher.add(source)) + } + }, + plugins: [ + base64ify(process.cwd()) // inline all url files + ] + }), + base64ify(process.cwd()), + postcssPresetEnv({browsers}) + ] + + if (minify) { + plugins.push(cssnano({ preset: 'default' })) + } + + plugins.push(postcssReporter({ clearMessages: true })) + + const transform = () => + postcss(plugins) + .process(`${configImport}${fs.readFileSync(entry, 'utf8')}`, { + from: undefined, + map: { inline: false }, + parser: postcssSafeParser, + to: outfile + }) + .then(function (results) { + if (outfile) { + mkdirp.sync(path.dirname(outfile)) + fs.writeFileSync(outfile, results.css) + if ("results.map") { + fs.writeFileSync(`${outfile}.map`, results.map.toString()) + } + if (watch) { + logger.log(`updated ${outfile}`) + } + } + return results + }) + + if (watch) { + watcher = chokidar.watch(entry) + watcher + .on('add', transform) + .on('change', transform) + .on('unlink', transform) + } + + return transform() +} + +const base64ify = postcss.plugin('postcss-base64ify', function () { + return function (css, result) { + css.replaceValues(/url\((\s*)(['"]?)(.+?)\2(\s*)\)/g, function (string) { + const filename = getUrl(string) + .split('?')[0] + .split('#')[0] + let file + if ( + filename.indexOf('data') === 0 || + filename.length === 0 || + filename.indexOf('http') === 0 + ) { + return string + } else if (filename[0] === '/') { + file = path.join(process.cwd(), filename) + } else { + const source = css.source.input.file + const dir = path.dirname(source) + file = path.resolve(dir, filename) + } + if (!fs.existsSync(file)) { + throw new Error(`File ${file} does not exist`) + } + const buffer = fs.readFileSync(file) + return ( + 'url("data:' + + mimeType.getType(filename) + + ';base64,' + + buffer.toString('base64') + + '")' + ) + }) + } +}) + +const URL_POSITION = 3 + +/** + * Extract the contents of a css url + * + * @param {string} value raw css + * @return {string} the contents of the url + */ +function getUrl (value) { + const reg = /url\((\s*)(['"]?)(.+?)\2(\s*)\)/g + const match = reg.exec(value) + const url = match[URL_POSITION] + return url +} diff --git a/lib/scenario-editor/utils/valhalla.js b/lib/scenario-editor/utils/valhalla.js index 408a8cec5..cdd2c177e 100644 --- a/lib/scenario-editor/utils/valhalla.js +++ b/lib/scenario-editor/utils/valhalla.js @@ -1,83 +1,25 @@ -// @flow - -import {isEqual as coordinatesAreEqual} from '@conveyal/lonlat' -import fetch from 'isomorphic-fetch' +import { isEqual as coordinatesAreEqual } from '@conveyal/lonlat' +import isomorphicFetch from 'isomorphic-fetch' //! Changed content import L from 'leaflet' -import {decode as decodePolyline} from 'polyline' +import { decode as decodePolyline } from 'polyline' import lineString from 'turf-linestring' import qs from 'qs' -// This can be used for logging line strings to geojson.io URLs for easy -// debugging. -// import {logCoordsToGeojsonio} from '../../editor/util/debug' - import { coordIsOutOfBounds } from '../../editor/util/map' -import type { - Coordinates, - LatLng -} from '../../types' - -type Instruction = { - distance: number, - heading: number, - interval: [number, number], - sign: number, - street_name: string, - text: string, - time: number -} - -type Path = { - ascend: number, - bbox: [number, number, number, number], - descend: number, - details: {}, - distance: number, - instructions: Array, - legs: [], - points: string, - points_encoded: boolean, - snapped_waypoints: string, - time: number, - transfers: number, - weight: number -} - -type GraphHopperResponse = { - hints: { - 'visited_nodes.average': string, - 'visited_nodes.sum': string - }, - info: { - copyrights: Array, - took: number - }, - paths: Array -} - -type GraphHopperAlternateServer = { - BBOX: Array, - KEY?: string, - URL?: string -} /** * Convert GraphHopper routing JSON response to polyline. */ -function handleGraphHopperRouting (path: Path, individualLegs: boolean = false): any { - const {instructions, points} = path +function handleGraphHopperRouting (path, individualLegs = false) { + // console.info({ path }); + const { instructions, points } = path // Decode polyline and reverse coordinates. - const decodedPolyline = decodePolyline(points).map(c => ([c[1], c[0]])) + const decodedPolyline = decodePolyline(points).map((c) => [c[1], c[0]]) if (individualLegs) { const segments = [] - // Keep track of the segment point intervals to split the line segment at. - // This appears to be the most reliable way to split up the geometry - // (previously distance was used here, but that provided inconstent results). + const segmentPointIndices = [0] - // Iterate over the instructions, accumulating segment point indices at each - // waypoint encountered. Indices are used to slice the line geometry when - // individual legs are needed. NOTE: Waypoint === routing point provided in - // the request. + instructions.forEach((instruction, i) => { if (instruction.text.match(/Waypoint (\d+)/) || i === instructions.length - 1) { segmentPointIndices.push(instruction.interval[0]) @@ -87,13 +29,11 @@ function handleGraphHopperRouting (path: Path, individualLegs: boolean = false): // at the provided indices. if (segmentPointIndices.length > 2) { for (var i = 1; i < segmentPointIndices.length; i++) { - // Get the indices of the points that the entire path should be sliced at - // Note: 'to' index is incremented by one because it is not inclusive. const [from, to] = [segmentPointIndices[i - 1], segmentPointIndices[i] + 1] const segment = decodedPolyline.slice(from, to) segments.push(segment) } - // console.log('individual legs', segments) + // console.log("individual legs", segments); return segments } else { // FIXME does this work for two input points? @@ -111,13 +51,11 @@ function handleGraphHopperRouting (path: Path, individualLegs: boolean = false): * distinct segments for each pair of points * @return {[type]} Array of coordinates or Array of arrays of coordinates. */ -export async function polyline ( - points: Array, - individualLegs?: boolean = false, - avoidMotorways?: boolean = false -): Promise { +export async function polyline (points, individualLegs = false, avoidMotorways = false) { + // console.info({ individualLegs }); let json const geometry = [] + const valhallaGeometry = [] //! New content try { // Chunk points into sets no larger than the max # of points allowed by // GraphHopper plan. @@ -134,41 +72,46 @@ export async function polyline ( const beginIndex = i + offset const endIndex = i + chunk + offset const chunkedPoints = points.slice(beginIndex, endIndex) - json = await routeWithGraphHopper(chunkedPoints, avoidMotorways) - const path = json && json.paths && json.paths[0] - // Route between chunked list of points - if (path) { - const result = handleGraphHopperRouting(path, individualLegs) - geometry.push(...result) - } else { - // If any of the routed legs fails, default to straight line (return null). - console.warn(`Error routing from point ${beginIndex} to ${endIndex}`, chunkedPoints) - return null + + //* The main function + const valhallaData = await getValhallaData(points, individualLegs, avoidMotorways) //! New content + valhallaGeometry.push(...valhallaData) //! New content + + //! Changed content + if (process.env.GRAPH_HOPPER_KEY) { + json = await routeWithGraphHopper(chunkedPoints, avoidMotorways) + + const path = json && json.paths && json.paths[0] + + if (path) { + const result = handleGraphHopperRouting(path, individualLegs) + geometry.push(...result) + } else { + // If any of the routed legs fails, default to straight line (return null). + console.warn(`Error routing from point ${beginIndex} to ${endIndex}`, chunkedPoints) + // return null; //! Changed content + } } count++ } - return geometry + // console.info("geometryGraphhopper:", geometry); //! New content + // console.info("valhallaGeometry:", valhallaGeometry); //! New content + // return geometry; //! Changed content + return valhallaGeometry //! New content } catch (e) { console.log(e) return null } } -export async function getSegment ( - points: Coordinates, - followRoad: boolean, - defaultToStraightLine: boolean = true, - avoidMotorways: boolean = false -): Promise { +export async function getSegment (points, followRoad, defaultToStraightLine = true, avoidMotorways = false) { // Store geometry to be returned here. + // console.info(2, "points:", points, { avoidMotorways }); let geometry if (followRoad) { // if snapping to streets, use routing service. const coordinates = await polyline( - points.map(p => ({lng: p[0], lat: p[1]})), + points.map((p) => ({ lng: p[0], lat: p[1] })), false, avoidMotorways ) @@ -201,12 +144,9 @@ export async function getSegment ( return geometry } -/** - * Call GraphHopper routing service with lat/lng coordinates. - * - * Example URL: https://graphhopper.com/api/1/route?point=49.932707,11.588051&point=50.3404,11.64705&vehicle=car&debug=true&&type=json - */ -export function routeWithGraphHopper (points: Array, avoidMotorways?: boolean): ?Promise { +// Fetch Data +export function routeWithGraphHopper (points, avoidMotorways) { + // console.info({ avoidMotorways }); if (points.length < 2) { console.warn('need at least two points to route with graphhopper', points) return null @@ -221,9 +161,9 @@ export function routeWithGraphHopper (points: Array, avoidMotorways?: bo if (process.env.GRAPH_HOPPER_ALTERNATES) { // $FlowFixMe This is a bit of a hack and now how env variables are supposed to work, but the yaml loader supports it. - const alternates: Array = process.env.GRAPH_HOPPER_ALTERNATES - alternates.forEach(alternative => { - const {BBOX} = alternative + const alternates = process.env.GRAPH_HOPPER_ALTERNATES + alternates.forEach((alternative) => { + const { BBOX } = alternative if (BBOX.length !== 4) { console.warn('Invalid BBOX for GRAPH_HOPPER_ALTERNATIVE') return @@ -238,10 +178,7 @@ export function routeWithGraphHopper (points: Array, avoidMotorways?: bo (point) => !coordIsOutOfBounds( point, - L.latLngBounds( - [alternative.BBOX[1], alternative.BBOX[0]], - [alternative.BBOX[3], alternative.BBOX[2]] - ) + L.latLngBounds([alternative.BBOX[1], alternative.BBOX[0]], [alternative.BBOX[3], alternative.BBOX[2]]) ) ) ) { @@ -261,30 +198,181 @@ export function routeWithGraphHopper (points: Array, avoidMotorways?: bo debug: true, type: 'json' } - const locations = points.map(p => (`point=${p.lat},${p.lng}`)).join('&') + const locations = points.map((p) => `point=${p.lat},${p.lng}`).join('&') // Avoiding motorways requires a POST request with a formatted body + //! Changed content const graphHopperRequest = avoidMotorways - ? fetch(`${graphHopperUrl}route?key=${params.key}`, - { - body: JSON.stringify({ - 'ch.disable': true, - // Custom model disincentives motorways - custom_model: { - 'priority': [{ - 'if': 'road_class == MOTORWAY', - 'multiply_by': 0.1 - }] - }, - debug: params.debug, - points: points.map(p => [p.lng, p.lat]), - profile: params.vehicle - }), - headers: { - 'Content-Type': 'application/json' + ? isomorphicFetch(`${graphHopperUrl}route?key=${params.key}`, { + body: JSON.stringify({ + 'ch.disable': true, + // Custom model disincentives motorways + custom_model: { + priority: [ + { + if: 'road_class == MOTORWAY', + multiply_by: 0.1 + } + ] }, - method: 'POST' - }) - : fetch(`${graphHopperUrl}route?${locations}&${qs.stringify(params)}`) + debug: params.debug, + points: points.map((p) => [p.lng, p.lat]), + profile: params.vehicle + }), + headers: { + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + : isomorphicFetch(`${graphHopperUrl}route?${locations}&${qs.stringify(params)}`) //! Changed content + + const response = graphHopperRequest.then((res) => res.json()) + return response +} + +//* --------------------------------------------------------------------------------------------------------------------------- +//! New content!!! +//* 1. Base function +const getValhallaData = async (points, individualLegs, avoidMotorways) => { + //* Temporally disabled removeDuplicates function + const shallRemoveDuplicates = false + + // console.info({ points, individualLegs, avoidMotorways }); + const encodedData = await fetchValhallaData(points, avoidMotorways) + // console.info("encodedData:", encodedData); + + const decodedData = encodedData.shapes.map((shape) => decodeValhallaPolyline(shape, 6)) + // console.info("decodedData:", decodedData); + + const decodedDataConverted = decodedData.map((data) => convertLatLon(data)) + // console.info("decodedDataConverted:", decodedDataConverted); + + const decodedDataConvertedFlat = shallRemoveDuplicates + ? removeDuplicates(decodedDataConverted.flat(1)) + : decodedDataConverted.flat(1) + // console.info("decodedDataConvertedFlat:", decodedDataConvertedFlat); + + const dataToReturn = individualLegs ? decodedDataConverted : decodedDataConvertedFlat + // console.info({ dataToReturn }); + return dataToReturn +} + +//* 2. Fetch Valhalla data +const fetchValhallaData = async (points, avoidMotorways) => { + // console.info("points:", points); + const preparedPoints = preparePoints(points) + // console.info("preparedPoints:", preparedPoints); + // console.info({ avoidMotorways }); + + const dataToFetch = { + locations: preparedPoints, + costing_options: { + auto: { + avoid_motorways: avoidMotorways + } + }, + costing: 'bus', + units: 'kilometers' + } + + const baseUrl = process.env.VALHALLA_URL + const params = { + json: JSON.stringify(dataToFetch) + } + + try { + //* V1 -> GET + const response = await fetch(`${baseUrl}?${new URLSearchParams(params)}`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + }) + //* V2 -> POST + // const response = await fetch(baseUrl, { + // method: "POST", // Change to POST + // headers: { + // "Content-Type": "application/json", + // }, + // body: JSON.stringify(dataToFetch), + // }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`) + } + + const data = await response.json() + const { + trip: { legs } + } = data + + const shapes = legs.map((leg) => leg.shape) + // console.info({ shapes }); + return { shapes } + } catch (error) { + console.error('Error fetching directions:', error) + return null + } +} + +//* 3. Prepare point data +const preparePoints = (points) => { + const reversedPoints = points.map((point) => { + return { lat: point.lat, lon: point.lng } + }) + return reversedPoints +} + +//* 4. Decode Valhalla Polyline string: https://valhalla.github.io/valhalla/decoding/#javascript +const decodeValhallaPolyline = (encoded, precision = 6) => { + const factor = Math.pow(10, precision) + let index = 0 + let lat = 0 + let lng = 0 + const coordinates = [] + + while (index < encoded.length) { + let result = 0 + let shift = 0 + let byte + + // Decode latitude + do { + byte = encoded.charCodeAt(index++) - 63 + result |= (byte & 0x1f) << shift + shift += 5 + } while (byte >= 0x20) + + const latitudeChange = result & 1 ? ~(result >> 1) : result >> 1 + lat += latitudeChange + + result = 0 + shift = 0 + + // Decode longitude + do { + byte = encoded.charCodeAt(index++) - 63 + result |= (byte & 0x1f) << shift + shift += 5 + } while (byte >= 0x20) + + const longitudeChange = result & 1 ? ~(result >> 1) : result >> 1 + lng += longitudeChange + + // Store the decoded coordinates + coordinates.push([lat / factor, lng / factor]) + } + + return coordinates +} + +//* 4. Remove duplicates in coordinates data +export const removeDuplicates = (dataToFilter) => { + const filteredData = Array.from(new Set(dataToFilter.map((elem) => JSON.stringify(elem)))).map((str) => JSON.parse(str)) + return filteredData +} - return graphHopperRequest.then(res => res.json()) +//* 5. Conversion [lat, lon] -> [lon, lat] and Vice Versa +const convertLatLon = (points) => { + const changedPoints = points.map((point) => [point[1], point[0]]) + return changedPoints }