Ajout géocodage

mbs-technologie · Apr 23, 2019 · ee4455d · ee4455d
1 parent e3ed000
commit ee4455d
Show file tree

Hide file tree

Showing 7 changed files with 103 additions and 6 deletions.
diff --git a/.env.sample b/.env.sample
@@ -0,0 +1 @@
+CADASTRE_COMMUNES_PATH=
diff --git a/.gitignore b/.gitignore
@@ -119,3 +119,4 @@ typings/
 # End of https://www.gitignore.io/api/node,macos
 
 /data
+/dist
diff --git a/improve-csv.js b/improve-csv.js
@@ -1,5 +1,6 @@
 #!/usr/bin/env node --max-old-space-size=4096
 /* eslint camelcase: off */
+require('dotenv').config()
 
 const {join} = require('path')
 const {createReadStream} = require('fs')
@@ -11,9 +12,12 @@ const {createGunzip} = require('gunzip-stream')
 const {groupBy} = require('lodash')
 const pumpify = require('pumpify').obj
 const getStream = require('get-stream')
+const centroid = require('@turf/centroid').default
+const truncate = require('@turf/truncate').default
 const {writeCsv} = require('./lib/csv')
 const {getCulturesMap, getCulturesSpecialesMap} = require('./lib/cultures')
 const {getDateMutation, getIdParcelle, getCodeCommune, getCodePostal} = require('./lib/parse')
+const {getParcellesCommune} = require('./lib/parcelles')
 const {getCodeDepartement} = require('./lib/util')
 
 function convertRow(row, {culturesMap, culturesSpecialesMap}) {
@@ -52,7 +56,9 @@ function convertRow(row, {culturesMap, culturesSpecialesMap}) {
     nature_culture: row['Nature culture'] in culturesMap ? culturesMap[row['Nature culture']] : '',
     code_nature_culture_speciale: row['Nature culture speciale'],
     nature_culture_speciale: row['Nature culture speciale'] in culturesSpecialesMap ? culturesSpecialesMap[row['Nature culture speciale']] : '',
-    surface_terrain: row['Surface terrain']
+    surface_terrain: row['Surface terrain'],
+    longitude: '',
+    latitude: ''
   }
 }
 
@@ -63,6 +69,10 @@ async function main() {
   const culturesSpecialesMap = await getCulturesSpecialesMap()
 
   await bluebird.each(millesimes, async millesime => {
+    console.log(`Millésime ${millesime}`)
+
+    console.log('Chargement des données')
+
     const rows = await getStream.array(pumpify(
       createReadStream(join(__dirname, 'data', `valeursfoncieres-${millesime}.txt.gz`)),
       createGunzip(),
@@ -72,21 +82,46 @@ async function main() {
       }})
     ))
 
+    /* Géocodage à la parcelle */
+
+    console.log('Géocodage à la parcelle')
+
+    const communesRows = groupBy(rows, 'code_commune')
+    await bluebird.map(Object.keys(communesRows), async codeCommune => {
+      const communeRows = communesRows[codeCommune]
+      const parcelles = await getParcellesCommune(codeCommune)
+
+      communeRows.forEach(row => {
+        if (parcelles && row.id_parcelle in parcelles) {
+          const parcelle = parcelles[row.id_parcelle]
+          const [lon, lat] = truncate(centroid(parcelle), {precision: 6}).geometry.coordinates
+          row.longitude = lon
+          row.latitude = lat
+        }
+      })
+    }, {concurrency: 8})
+
+    /* Export des données à la commune */
+
+    console.log('Export des données à la commune')
+
     const communesGroupedRows = groupBy(rows, 'code_commune')
 
     await bluebird.map(Object.keys(communesGroupedRows), async codeCommune => {
-      console.log(codeCommune)
       const communeRows = communesGroupedRows[codeCommune]
       const codeDepartement = getCodeDepartement(codeCommune)
       const departementPath = join(__dirname, 'dist', millesime, 'communes', codeDepartement)
       await ensureDir(departementPath)
       await writeCsv(join(departementPath, `${codeCommune}.csv`), communeRows)
     }, {concurrency: 8})
 
+    /* Export des données au département */
+
+    console.log('Export des données au département')
+
     const departementsGroupedRows = groupBy(rows, 'code_departement')
 
     await bluebird.map(Object.keys(departementsGroupedRows), async codeDepartement => {
-      console.log(codeDepartement)
       const departementRows = departementsGroupedRows[codeDepartement]
       const departementsPath = join(__dirname, 'dist', millesime, 'departements')
       await ensureDir(departementsPath)

diff --git a/lib/csv.js b/lib/csv.js
@@ -1,5 +1,5 @@
 const {promisify} = require('util')
-const finished = promisify(require('stream').finished)
+const pipeline = promisify(require('stream').pipeline)
 const {PassThrough} = require('stream')
 const {createGzip} = require('zlib')
 const {createReadStream, createWriteStream} = require('fs')
@@ -17,12 +17,12 @@ function readCsv(filePath, options = {}) {
 }
 
 function writeCsv(filePath, rows, options = {}) {
-  return finished(pumpify(
+  return pipeline(
     intoStream.object(rows),
     csvWriter({separator: options.separator || ','}),
     filePath.endsWith('.gz') ? createGzip() : new PassThrough(),
     createWriteStream(filePath)
-  ))
+  )
 }
 
 module.exports = {readCsv, writeCsv}
diff --git a/lib/parcelles.js b/lib/parcelles.js
@@ -0,0 +1,24 @@
+const {promisify} = require('util')
+const {join} = require('path')
+const gunzip = promisify(require('zlib').gunzip)
+const {readFile, pathExists} = require('fs-extra')
+const {keyBy} = require('lodash')
+const {getCodeDepartement} = require('./util')
+
+const communesPath = process.env.CADASTRE_COMMUNES_PATH
+
+async function getParcellesCommune(codeCommune) {
+  const filePath = join(communesPath, getCodeDepartement(codeCommune), codeCommune, `cadastre-${codeCommune}-parcelles.json.gz`)
+
+  if (!(await pathExists(filePath))) {
+    return
+  }
+
+  const gzippedFileContent = await readFile(filePath)
+  const fileContent = await gunzip(gzippedFileContent)
+  const featureCollection = JSON.parse(fileContent)
+
+  return keyBy(featureCollection.features, 'properties.id')
+}
+
+module.exports = {getParcellesCommune}
diff --git a/package.json b/package.json
@@ -12,9 +12,12 @@
     "improve-csv": "node --max-old-space-size=4096 improve-csv"
   },
   "dependencies": {
+    "@turf/centroid": "^6.0.2",
+    "@turf/truncate": "^6.0.1",
     "bluebird": "^3.5.4",
     "csv-parser": "^2.2.0",
     "csv-write-stream": "^2.0.0",
+    "dotenv": "^7.0.0",
     "fs-extra": "^7.0.1",
     "get-stream": "^5.1.0",
     "gunzip-stream": "^1.0.1",

diff --git a/yarn.lock b/yarn.lock
@@ -31,6 +31,34 @@
   resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b"
   integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==
 
+"@turf/centroid@^6.0.2":
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/@turf/centroid/-/centroid-6.0.2.tgz#c4eb16b4bc60b692f74e1809cf9a7c4a4f5ba1cc"
+  integrity sha512-auyDauOtC4eddH7GC3CHFTDu2PKhpSeKCRhwhHhXtJqn2dWCJQNIoCeJRmfXRIbzCWhWvgvQafvvhq8HNvmvWw==
+  dependencies:
+    "@turf/helpers" "6.x"
+    "@turf/meta" "6.x"
+
+"@turf/helpers@6.x":
+  version "6.1.4"
+  resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.1.4.tgz#d6fd7ebe6782dd9c87dca5559bda5c48ae4c3836"
+  integrity sha512-vJvrdOZy1ngC7r3MDA7zIGSoIgyrkWcGnNIEaqn/APmw+bVLF2gAW7HIsdTxd12s5wQMqEpqIQrmrbRRZ0xC7g==
+
+"@turf/meta@6.x":
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/@turf/meta/-/meta-6.0.2.tgz#eb92951126d24a613ac1b7b99d733fcc20fd30cf"
+  integrity sha512-VA7HJkx7qF1l3+GNGkDVn2oXy4+QoLP6LktXAaZKjuT1JI0YESat7quUkbCMy4zP9lAUuvS4YMslLyTtr919FA==
+  dependencies:
+    "@turf/helpers" "6.x"
+
+"@turf/truncate@^6.0.1":
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/@turf/truncate/-/truncate-6.0.1.tgz#b052255ad17a114e50d2f77d9339454918e750b7"
+  integrity sha512-0n6MauRHIhnlNPKnVa65YUPNh0AExDmBfvg7v5w1corBlYHZXLew3rS9IKw7o6VqHaMUQs5RT/b3/77RlTetJg==
+  dependencies:
+    "@turf/helpers" "6.x"
+    "@turf/meta" "6.x"
+
 "@types/events@*":
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
@@ -619,6 +647,11 @@ dot-prop@^4.1.0:
   dependencies:
     is-obj "^1.0.0"
 
+dotenv@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-7.0.0.tgz#a2be3cd52736673206e8a85fb5210eea29628e7c"
+  integrity sha512-M3NhsLbV1i6HuGzBUH8vXrtxOk+tWmzWKDMbAVSUp3Zsjm7ywFeuwrUXhmhQyRK1q5B5GGy7hcXPbj3bnfZg2g==
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"