segmentio · jedrzejchalubek · Oct 24, 2019 · Oct 10, 2019 · Oct 23, 2019 · Oct 23, 2019
diff --git a/package.json b/package.json
@@ -12,11 +12,17 @@
     "@babel/plugin-proposal-object-rest-spread": "7.5.5",
     "@babel/plugin-transform-runtime": "7.6.0",
     "@babel/preset-env": "7.6.0",
+    "axios": "0.19.0",
     "babel-loader": "8.0.6",
     "concurrently": "4.1.2",
+    "front-matter": "3.0.2",
+    "glob": "7.1.4",
+    "js-yaml": "3.13.1",
+    "superagent": "5.1.0",
     "webpack": "4.40.2",
     "webpack-cli": "3.3.9"
   },
   "dependencies": {
+    "dotenv": "8.2.0"
   }
 }
diff --git a/scripts/catalog.js b/scripts/catalog.js
@@ -0,0 +1,236 @@
+const axios = require('axios');
+const path = require('path');
+const fs = require('fs');
+const yaml = require('js-yaml');
+
+require('dotenv').config();
+
+PLATFORM_API_URL = "https://platform.segmentapis.com"
+
+const slugify = (displayName) => {
+  let slug = displayName.toLowerCase().replace(/\s+/g, '-')
+  if (slug === '.net') slug = 'net'
+  if (slug === 'roku-(alpha)') slug = 'roku'
+  return slug
+}
+
+const getCatalog = async (url, page_token = "") => {
+  try {
+   const res = await axios.get(url, {
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${process.env.PLATFORM_API_TOKEN}`
+      },
+      params: {
+        page_token,
+        page_size: 100
+      }
+    });
+    return res.data
+  } catch (error) {
+    console.log(error)
+  }
+}
+
+const updateDestinationCategories = async () => {
+  let destinations = []
+  let nextPageToken = null
+  let categories = new Set();
+  let destinationCategories = []
+
+  while (nextPageToken !== "") {
+    const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/destinations`, nextPageToken)
+    destinations = destinations.concat(res.destinations)
+    nextPageToken = res.next_page_token
+  }
+  destinations.forEach(destination => {
+    let temp = [destination.categories.primary, destination.categories.secondary, ...destination.categories.additional]
+    temp = temp.filter(category => category != '')
+    temp.reduce((s, e) => s.add(e), categories);
+  })
+
+  const destinationArray = Array.from(categories)
+  destinationArray.forEach(category => {
+
+    destinationCategories.push({
+      display_name: category,
+      slug: slugify(category)
+    })
+  })
+  const options = { noArrayIndent: true };
+  let output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
+  output += yaml.safeDump({ items: destinationCategories }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalogV2/destination_categories.yml`), output);
+}
+
+const updateSourceCategories = async () => {
+  let sources = []
+  let nextPageToken = null
+  let categories = new Set();
+  let sourceCategories = []
+
+  while (nextPageToken !== "") {
+    const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/sources`, nextPageToken)
+    sources = sources.concat(res.sources)
+    nextPageToken = res.next_page_token
+  }
+  sources.forEach(source => {
+    source.categories.reduce((s, e) => s.add(e), categories);
+  })
+
+  const sourceArray = Array.from(categories)
+  sourceArray.forEach(category => {
+
+    sourceCategories.push({
+      display_name: category,
+      slug: slugify(category)
+    })
+  })
+  const options = { noArrayIndent: true };
+  let output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
+  output += yaml.safeDump({ items: sourceCategories }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalogV2/source_categories.yml`), output);
+}
+
+updateSourceCategories()
+updateDestinationCategories()
+
+const updateSources = async () => {
+  let sources = []
+  let nextPageToken = null
+  while (nextPageToken !== "") {
+    const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/sources`, nextPageToken)
+    sources = sources.concat(res.sources)
+    nextPageToken = res.next_page_token
+  }
+  sources.sort((a, b) => {
+    if(a.display_name < b.display_name) { return -1; }
+    if(a.display_name > b.display_name) { return 1; }
+    return 0;
+  })
+  const options = { noArrayIndent: true };
+  let output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
+  output += yaml.safeDump({ sections: sources }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalogV2/sources.yml`), output);
+}
+
+const updateSourcesV2 = async () => {
+  let sources = []
+  let sourcesUpdated = []
+  let nextPageToken = null
+
+  while (nextPageToken !== "") {
+    const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/sources`, nextPageToken)
+    sources = sources.concat(res.sources)
+    nextPageToken = res.next_page_token
+  }
+  sources.sort((a, b) => {
+    if(a.display_name < b.display_name) { return -1; }
+    if(a.display_name > b.display_name) { return 1; }
+    return 0;
+  })
+  const libraryCategories = [
+    'server',
+    'mobile',
+    'ott',
+    'roku',
+    'website'
+  ]
+  sources.forEach(source => {
+    let slug = slugify(source.display_name)
+
+    let url = ''
+    let mainCategory = source.categories[0] ? source.categories[0].toLowerCase() : ''
+    if (libraryCategories.includes(mainCategory)) {
+      url = `connections/sources/catalog/libraries/${mainCategory}/${slug}`
+    } else {
+      url = `connections/sources/catalog/cloud-apps/${slug}`
+    }
+    let updatedSource = {
+      display_name: source.display_name,
+      slug,
+      name: source.name,
+      description: source.description,
+      url,
+      logo: {
+        url: source.logos.logo
+      },
+      mark: {
+        url: source.logos.mark
+      },
+      categories: source.categories
+    }
+    sourcesUpdated.push(updatedSource)
+  })
+  const options = { noArrayIndent: true };
+  let output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
+  output += yaml.safeDump({ items: sourcesUpdated }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalogV2/sources.yml`), output);
+}
+
+const updateDestinations = async () => {
+  let destinations = []
+  let nextPageToken = null
+  while (nextPageToken !== "") {
+    const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/destinations`, nextPageToken)
+    destinations = destinations.concat(res.destinations)
+    nextPageToken = res.next_page_token
+  }
+  destinations.sort((a, b) => {
+    if(a.display_name < b.display_name) { return -1; }
+    if(a.display_name > b.display_name) { return 1; }
+    return 0;
+  })
+  const options = { noArrayIndent: true };
+  let output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
+  output += yaml.safeDump({ items: destinations }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalogV2/destinations.yml`), output);
+}
+
+const updateDestinationsV2 = async () => {
+  let destinations = []
+  let destinationsUpdated = []
+  let nextPageToken = null
+  while (nextPageToken !== "") {
+    const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/destinations`, nextPageToken)
+    destinations = destinations.concat(res.destinations)
+    nextPageToken = res.next_page_token
+  }
+  destinations.sort((a, b) => {
+    if(a.display_name < b.display_name) { return -1; }
+    if(a.display_name > b.display_name) { return 1; }
+    return 0;
+  })
+  destinations.forEach(destination => {
+    let slug = slugify(destination.display_name)
+
+    let categories = [destination.categories.primary, destination.categories.secondary, ...destination.categories.additional]
+    categories = categories.filter(category => category != '')
+
+    let updatedDestination = {
+      display_name: destination.display_name,
+      slug,
+      name: destination.name,
+      description: destination.description,
+      url: `connections/destinations/catalog/${slug}`,
+      status: destination.status,
+      logo:  {
+        url: destination.logos.logo
+      },
+      mark: {
+        url: destination.logos.mark
+      },
+      categories
+    }
+    destinationsUpdated.push(updatedDestination)
+  })
+  const options = { noArrayIndent: true };
+  let output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
+  output += yaml.safeDump({ items: destinationsUpdated }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalogV2/destinations.yml`), output);
+}
+
+// updateSources()
+// updateSourcesV2()
+// updateDestinations()
+// updateDestinationsV2()
diff --git a/scripts/nav.js b/scripts/nav.js
@@ -0,0 +1,79 @@
+const glob = require('glob');
+const path = require('path');
+const fs = require('fs');
+const fm = require('front-matter');
+const yaml = require('js-yaml');
+
+const ignore = [ 'connections/**', '_*/**', '*.md' ];
+
+const capitalize = (str) => str.charAt(0).toLocaleUpperCase() + str.slice(1);
+const getTitle = (str) => str.split('-').map(capitalize).join(' ');
+
+const getSection = (accum, paths) => {
+    const slug = paths.join('/');
+    if (accum[slug]) return accum[slug];
+
+    const section_title = getTitle(paths[paths.length - 1]);
+    const section = [];
+
+    const subsection = paths.length > 1
+        ? { section_title, slug, section }
+        : { section_title, section };
+
+    accum[slug] = subsection;
+    if (paths.length > 1) getSection(accum, paths.slice(0, -1)).section.push(subsection);
+
+    return subsection;
+};
+
+const files = glob.sync('**/*.md', { ignore, cwd: path.resolve(__dirname, '../src') });
+
+const sections = files.reduce((accum, file) => {
+    const paths = file.split('/');
+
+    // Not a valid path or some deep-nested directory structure we don't support
+    if (paths.length < 2 || paths.length > 3) {
+        console.log(`skipping ${file}`);
+        return accum;
+    }
+
+    // read in the .md file, parse the frontmatter attributes
+    const f = fm(fs.readFileSync(path.resolve(__dirname, '../src', file), 'utf8'));
+
+    const title = f.attributes.title || getTitle(paths[paths.length - 1].slice(0, -3));
+    const s = getSection(accum, paths.slice(0, -1));
+
+    if (paths[paths.length - 1] === 'index.md') {
+        s.section.unshift({ path: `/${paths.slice(0, -1).join('/')}`, title });
+    } else {
+        s.section.push({ path: `/${paths.join('/').slice(0, -3)}`, title });
+    }
+
+    return accum;
+}, {});
+
+const mainSections = {};
+const legalSections = {};
+const apiSections = {};
+const partnerSections = {};
+
+Object.keys(sections).filter(key => !key.includes('/')).forEach((key) => {
+    const value = sections[key];
+
+    switch (key) {
+        case 'legal': legalSections[key] = value; break;
+        case 'api': apiSections[key] = value; break;
+        case 'partners': partnerSections[key] = value; break;
+        default: mainSections[key] = value; break;
+    }
+});
+
+[   ['main', mainSections],
+    ['legal', legalSections],
+    ['api', apiSections],
+    ['partners', partnerSections]
+].forEach(([ name, sections ]) => {
+    const options = { noArrayIndent: true };
+    const output = yaml.safeDump({ sections: Object.values(sections) }, options);
+    fs.writeFileSync(path.resolve(__dirname, `../src/_data/sidenav1/${name}.yml`), output);
+});