feat(CO2): implement carbon footprint data in cloud pricing api

README.md

@@ -40,7 +40,7 @@ The pricing DB dump is downloaded from Infracost's API as that simplifies the ta
 
 ## Deployment
 
-It should take around 15 mins to deploy the Cloud Pricing API. Two deployment methods are supported:
+Two deployment methods are supported:
 1. If you have a Kubernetes cluster, we recommend using [our Helm Chart](https://github.com/infracost/helm-charts/tree/master/charts/cloud-pricing-api).
 2. If you prefer to deploy in a VM, we recommend using [Docker compose](#docker-compose).
 
@@ -136,7 +136,7 @@ See [the Infracost docs](https://www.infracost.io/docs/cloud_pricing_api/self_ho
 
 ## Troubleshooting
 
-Please see [this section](https://www.infracost.io/docs/cloud_pricing_api/self_hosted/#troubleshooting) and join our [community Slack channel](https://www.infracost.io/community-chat), we'll help you very quickly 😄
+Please see [this section](https://www.infracost.io/docs/cloud_pricing_api/self_hosted/#troubleshooting).
 
 ## Contributing
 

package.json

@@ -13,6 +13,7 @@
     "data:dump": "node dist/cmd/dataDump.js",
     "data:download": "node dist/cmd/dataDownload.js",
     "data:load": "node dist/cmd/dataLoad.js",
+    "data:patchEmissions": "node dist/cmd/dataPatchEmissions.js",
     "data:status": "node dist/cmd/dataStatus.js",
     "job:init": "npm run db:setup && npm run job:update",
     "job:update": "npm run data:download && npm run data:load",
@@ -21,6 +22,7 @@
     "data:dump:dev": "ts-node src/cmd/dataDump.ts",
     "data:download:dev": "ts-node src/cmd/dataDownload.ts",
     "data:load:dev": "ts-node src/cmd/dataLoad.ts",
+    "data:patchEmissions:dev": "ts-node src/cmd/dataPatchEmissions.ts",
     "data:status:dev": "ts-node src/cmd/dataStatus.ts",
     "job:init:dev": "npm run db:setup:dev && npm run job:update:dev",
     "job:update:dev": "npm run data:download:dev && npm run data:load:dev",

src/cmd/dataPatchEmissions.ts

@@ -0,0 +1,66 @@
+import ProgressBar from 'progress';
+import config from '../config';
+import emissionsCoef from '../data/emissions/coefficients.json';
+import { PoolClient } from 'pg';
+
+async function run(): Promise<void> {
+  const pool = await config.pg();
+
+  const client = await pool.connect();
+
+  const progressBar = new ProgressBar(
+    '-> loading [:bar] :percent (:etas remaining)',
+    {
+      width: 40,
+      complete: '=',
+      incomplete: ' ',
+      renderThrottle: 500,
+      total: Object.keys(emissionsCoef).length,
+    }
+  );
+
+  // For each coefficient in the JSON file, update the corresponding products.
+  config.logger.info('Updating product emissions');
+
+  for (const [key, coefficient] of Object.entries(emissionsCoef)) {
+    const [_, skuId, region] = key.split('_');
+    await updateProductEmissions(client, skuId, region, coefficient);
+    progressBar.tick();
+  }
+}
+
+const updateProductEmissions = async (
+  client: PoolClient,
+  skuId: string,
+  region: string,
+  coefficient: number
+): Promise<void> => {
+  const emissionData = JSON.stringify([
+    {
+      emissionHash: 'emissionHash',
+      unit: 'kgCO2e',
+      emissions: coefficient,
+      startUsageAmount: 0,
+    },
+  ]);
+
+  await client.query(
+    `
+      UPDATE "products"
+      SET "emissions" = $1
+      WHERE "sku" = $2 AND "region" = $3
+      `,
+    [emissionData, skuId, region]
+  );
+};
+
+config.logger.info('Starting: loading data into DB');
+run()
+  .then(() => {
+    config.logger.info('Completed: loading data into DB');
+    process.exit(0);
+  })
+  .catch((err) => {
+    config.logger.error(err);
+    process.exit(1);
+  });

src/db/query.ts

@@ -1,6 +1,6 @@
 // In order to make upserting more efficient, prices in postgres are stored as a map of priceHash -> prices.
 import format from 'pg-format';
-import { Price, Product, ProductAttributes } from './types';
+import { Emission, Price, Product, ProductAttributes } from './types';
 import config from '../config';
 
 type AttributeFilter = {
@@ -10,7 +10,7 @@ type AttributeFilter = {
   value_regex?: string;
 };
 
-type ProductWithPriceMap = {
+type ProductWithPriceAndEmissionsMap = {
   productHash: string;
   sku: string;
   vendorName: string;
@@ -19,10 +19,11 @@ type ProductWithPriceMap = {
   productFamily: string;
   attributes: ProductAttributes;
   prices: { [priceHash: string]: Price[] };
+  emissions: { [emissionHash: string]: Emission[] };
 };
 
-function flattenPrices(p: ProductWithPriceMap): Product {
-  return { ...p, prices: Object.values(p.prices).flat() };
+function flattenPrices(p: ProductWithPriceAndEmissionsMap): Product {
+  return { ...p, prices: Object.values(p.prices).flat(), emissions: Object.values(p.emissions).flat() };
 }
 
 // eslint-disable-next-line import/prefer-default-export
@@ -53,7 +54,7 @@ export async function findProducts(
   }
 
   const response = await pool.query(sql);
-  const products = response.rows as ProductWithPriceMap[];
+  const products = response.rows as ProductWithPriceAndEmissionsMap[];
   return products.map((product) => flattenPrices(product));
 }
 

src/db/setup.ts

@@ -18,9 +18,10 @@ export async function createProductsTable(
         service text NOT NULL,
         "productFamily" text DEFAULT ''::text NOT NULL,
         attributes jsonb NOT NULL,
-        prices jsonb NOT NULL, 
+        prices jsonb NOT NULL,
+        emissions jsonb NOT NULL DEFAULT '[]'::jsonb,
         CONSTRAINT %I PRIMARY KEY("productHash")
-      )   
+      )
     `,
       tableName,
       `${tableName}_pkey`
@@ -72,7 +73,7 @@ export async function createStatsTable(
         total_runs bigint DEFAULT 0,
         ci_runs bigint DEFAULT 0,
         non_ci_runs bigint DEFAULT 0
-      )   
+      )
     `,
       tableName
     )
@@ -101,7 +102,7 @@ export async function createInstallsTable(
       (
         install_id uuid PRIMARY KEY NOT NULL,
         created_at timestamp DEFAUlT NOW() NOT NULL
-      )   
+      )
     `,
       tableName
     )

src/db/types.ts

@@ -9,6 +9,7 @@ export type Product = {
   productFamily: string;
   attributes: ProductAttributes;
   prices: Price[];
+  emissions: Emission[];
 };
 
 export type Price = {
@@ -26,3 +27,14 @@ export type Price = {
   termOfferingClass?: string;
   description?: string;
 };
+
+export type Emission = {
+  emissionHash: string;
+  unit: string;
+  emissions: string;
+  effectiveDateStart: string;
+  effectiveDateEnd?: string;
+  startUsageAmount?: string;
+  endUsageAmount?: string;
+  description?: string;
+}

src/resolvers.ts

@@ -1,6 +1,6 @@
 import { IResolvers } from '@graphql-tools/utils';
 import mingo from 'mingo';
-import { Price, Product } from './db/types';
+import { Emission, Price, Product } from './db/types';
 import currency, { CURRENCY_CODES } from './utils/currency';
 import { findProducts } from './db/query';
 import { ApplicationOptions } from './app';
@@ -28,6 +28,10 @@ interface PricesArgs {
   filter: Filter;
 }
 
+interface EmissionsArgs {
+  filter: Filter;
+}
+
 type TransformedProductAttribute = {
   key: string;
   value: string;
@@ -77,6 +81,14 @@ const getResolvers = <TContext>(
 
       return prices;
     },
+    emissions: async (product: Product, args: EmissionsArgs): Promise<Emission[]> => {
+
+      const emissions = mingo
+        .find(product.emissions, transformFilter(args.filter))
+        .all() as Emission[];
+
+        return emissions;
+    },
   },
   Price:
     // For every alternate currency, add a resolver that converts from USD.

src/scrapers/awsBulk.ts

@@ -207,6 +207,7 @@ function parseProduct(productJson: ProductJson) {
     sku: productJson.sku,
     attributes: productJson.attributes,
     prices: [],
+    emissions: [],
   };
 
   product.productHash = generateProductHash(product);

src/scrapers/azureRetail.ts

@@ -139,6 +139,7 @@ function parseProduct(productJson: ProductJson): Product {
       meterName: productJson.meterName,
     },
     prices: [],
+    emissions: [],
   };
 
   product.productHash = generateProductHash(product);

src/scrapers/gcpCatalog.ts

@@ -191,6 +191,7 @@ function parseProduct(productJson: ProductJson, region: string): Product {
       resourceGroup: productJson.category.resourceGroup,
     },
     prices: [],
+    emissions: [],
   };
 
   product.productHash = generateProductHash(product);

src/scrapers/gcpMachineTypes.ts

@@ -129,6 +129,7 @@ async function scrape(): Promise<void> {
           machineType: name,
         },
         prices: [],
+        emissions: [],
       };
 
       product.productHash = generateProductHash(product);

src/typeDefs.ts

@@ -18,6 +18,17 @@ const typeDefs = gql`
     termOfferingClass: String
   }
 
+  type Emission {
+    emissionHash: String!
+    unit: String!
+    emissions: String!
+    effectiveDateStart: String
+    effectiveDateEnd: String
+    startUsageAmount: String
+    endUsageAmount: String
+    description: String
+  }
+
   type Product {
     productHash: String!
     vendorName: String!
@@ -27,6 +38,7 @@ const typeDefs = gql`
     sku: String!
     attributes: [Attribute]
     prices(filter: PriceFilter): [Price]
+    emissions(filter: EmissionFilter): [Emission]
   }
 
   type Attribute {
@@ -61,6 +73,14 @@ const typeDefs = gql`
     termOfferingClass: String
   }
 
+  input EmissionFilter {
+    unit: String
+    description: String
+    description_regex: String
+    startUsageAmount: String
+    endUsageAmount: String
+  }
+
   type Query {
     products(filter: ProductFilter): [Product]
   }

tsconfig.json

@@ -6,6 +6,7 @@
     "lib": ["esnext"],
     "esModuleInterop": true,
     "declaration": true,
+    "resolveJsonModule": true,
   },
   "include": [
     "src/**/*"