Added optional support for remote refs resolution

Vishal Shingala · Vishal Shingala · commit f202cb29910b · 2020-09-25T01:30:52.000+05:30
diff --git a/lib/options.js b/lib/options.js
@@ -107,6 +107,24 @@ module.exports = {
         external: true,
         usage: ['CONVERSION']
       },
+      {
+        name: 'Resolve remote references',
+        id: 'resolveRemoteRefs',
+        type: 'boolean',
+        default: false,
+        description: 'Select whether to resolve remote references.',
+        external: true,
+        usage: ['CONVERSION']
+      },
+      {
+        name: 'Source URL of definition',
+        id: 'sourceUrl',
+        type: 'string',
+        default: '',
+        description: 'Specify source URL of definition to resolve remote references mentioned in it.',
+        external: true,
+        usage: ['CONVERSION']
+      },
       {
         name: 'Enable Schema Faking',
         id: 'schemaFaker',
diff --git a/lib/parse.js b/lib/parse.js
@@ -177,12 +177,40 @@ module.exports = {
       cache: [],
       externals: [],
       externalRefs: {},
+      ignoreIOErrors: true,
       rewriteRefs: true,
       openapi: openapi,
       files: files
     });
   },
 
+  /**
+   * Resolves remote and URL $ref from OpenAPI definition based on given options
+   *
+   * @param {*} openapi - OpenAPI definition
+   * @param {*} options - options
+   * @param {*} cb - callback function
+   * @return {*} - err if present
+   */
+  resolveRemoteRefs: function (openapi, options, cb) {
+    if (options.resolveRemoteRefs) {
+      return resolver.resolve(openapi, options.sourceUrl, {
+        resolve: true,
+        externals: [],
+        externalRefs: {},
+        ignoreIOErrors: true,
+        openapi: openapi
+      })
+        .then(function() {
+          return cb(null);
+        })
+        .catch(function(err) {
+          return cb(err);
+        });
+    }
+    return cb(null);
+  },
+
   /** Resolves all OpenAPI file references and returns a single OAS Object
    *
    * @param {Object} source Root file path
diff --git a/lib/schemapack.js b/lib/schemapack.js
@@ -237,90 +237,99 @@ class SchemaPack {
       return callback(new OpenApiErr('The schema must be validated before attempting conversion'));
     }
 
-    // this cannot be attempted before validation
-    componentsAndPaths = {
-      components: this.openapi.components,
-      paths: this.openapi.paths
-    };
+    parse.resolveRemoteRefs(this.openapi, options, (err) => {
+      if (err) {
+        return callback(null, {
+          result: false,
+          reason: err.name + ': ' + err.message
+        });
+      }
 
-    // create and sanitize basic spec
-    openapi = this.openapi;
-    openapi.servers = _.isEmpty(openapi.servers) ? [{ url: '/' }] : openapi.servers;
-    openapi.securityDefs = _.get(openapi, 'components.securitySchemes', {});
-    openapi.baseUrl = _.get(openapi, 'servers.0.url', '{{baseURL}}');
+      // this cannot be attempted before validation
+      componentsAndPaths = {
+        components: this.openapi.components,
+        paths: this.openapi.paths
+      };
 
-    // TODO: Multiple server variables need to be saved as environments
-    openapi.baseUrlVariables = _.get(openapi, 'servers.0.variables');
+      // create and sanitize basic spec
+      openapi = this.openapi;
+      openapi.servers = _.isEmpty(openapi.servers) ? [{ url: '/' }] : openapi.servers;
+      openapi.securityDefs = _.get(openapi, 'components.securitySchemes', {});
+      openapi.baseUrl = _.get(openapi, 'servers.0.url', '{{baseURL}}');
 
-    // Fix {scheme} and {path} vars in the URL to :scheme and :path
-    openapi.baseUrl = schemaUtils.fixPathVariablesInUrl(openapi.baseUrl);
+      // TODO: Multiple server variables need to be saved as environments
+      openapi.baseUrlVariables = _.get(openapi, 'servers.0.variables');
 
-    // Creating a new instance of a Postman collection
-    // All generated folders and requests will go inside this
-    generatedStore.collection = new sdk.Collection({
-      info: {
-        name: _.get(openapi, 'info.title', COLLECTION_NAME)
-      }
-    });
+      // Fix {scheme} and {path} vars in the URL to :scheme and :path
+      openapi.baseUrl = schemaUtils.fixPathVariablesInUrl(openapi.baseUrl);
+
+      // Creating a new instance of a Postman collection
+      // All generated folders and requests will go inside this
+      generatedStore.collection = new sdk.Collection({
+        info: {
+          name: _.get(openapi, 'info.title', COLLECTION_NAME)
+        }
+      });
 
-    if (openapi.security) {
-      authHelper = schemaUtils.getAuthHelper(openapi, openapi.security);
-      if (authHelper) {
-        generatedStore.collection.auth = authHelper;
+      if (openapi.security) {
+        authHelper = schemaUtils.getAuthHelper(openapi, openapi.security);
+        if (authHelper) {
+          generatedStore.collection.auth = authHelper;
+        }
       }
-    }
-    // ---- Collection Variables ----
-    // adding the collection variables for all the necessary root level variables
-    // and adding them to the collection variables
-    schemaUtils.convertToPmCollectionVariables(
-      openapi.baseUrlVariables,
-      'baseUrl',
-      openapi.baseUrl
-    ).forEach((element) => {
-      generatedStore.collection.variables.add(element);
-    });
+      // ---- Collection Variables ----
+      // adding the collection variables for all the necessary root level variables
+      // and adding them to the collection variables
+      schemaUtils.convertToPmCollectionVariables(
+        openapi.baseUrlVariables,
+        'baseUrl',
+        openapi.baseUrl
+      ).forEach((element) => {
+        generatedStore.collection.variables.add(element);
+      });
 
-    generatedStore.collection.describe(schemaUtils.getCollectionDescription(openapi));
+      generatedStore.collection.describe(schemaUtils.getCollectionDescription(openapi));
 
-    // Only change the stack limit if the optimizeConversion option is true
-    if (options.optimizeConversion) {
-      // Deciding stack limit based on size of the schema, number of refs and number of paths.
-      analysis = schemaUtils.analyzeSpec(openapi);
+      // Only change the stack limit if the optimizeConversion option is true
+      if (options.optimizeConversion) {
+        // Deciding stack limit based on size of the schema, number of refs and number of paths.
+        analysis = schemaUtils.analyzeSpec(openapi);
 
-      // Update options on the basis of analysis.
-      options = schemaUtils.determineOptions(analysis, options);
-    }
+        // Update options on the basis of analysis.
+        options = schemaUtils.determineOptions(analysis, options);
+      }
 
 
-    // ---- Collection Items ----
-    // Adding the collection items from openapi spec based on folderStrategy option
-    // For tags, All operations are grouped based on respective tags object
-    // For paths, All operations are grouped based on corresponding paths
-    try {
-      if (options.folderStrategy === 'tags') {
-        schemaUtils.addCollectionItemsUsingTags(openapi, generatedStore, componentsAndPaths, options, schemaCache);
+      // ---- Collection Items ----
+      // Adding the collection items from openapi spec based on folderStrategy option
+      // For tags, All operations are grouped based on respective tags object
+      // For paths, All operations are grouped based on corresponding paths
+      try {
+        if (options.folderStrategy === 'tags') {
+          schemaUtils.addCollectionItemsUsingTags(openapi, generatedStore, componentsAndPaths, options, schemaCache);
+        }
+        else {
+          schemaUtils.addCollectionItemsUsingPaths(openapi, generatedStore, componentsAndPaths, options, schemaCache);
+        }
       }
-      else {
-        schemaUtils.addCollectionItemsUsingPaths(openapi, generatedStore, componentsAndPaths, options, schemaCache);
+      catch (e) {
+        return callback(e);
       }
-    }
-    catch (e) {
-      return callback(e);
-    }
 
-    collectionJSON = generatedStore.collection.toJSON();
+      collectionJSON = generatedStore.collection.toJSON();
 
-    // this needs to be deleted as even if version is not specified to sdk,
-    // it returns a version property with value set as undefined
-    // this fails validation against v2.1 collection schema definition.
-    delete collectionJSON.info.version;
+      // this needs to be deleted as even if version is not specified to sdk,
+      // it returns a version property with value set as undefined
+      // this fails validation against v2.1 collection schema definition.
+      delete collectionJSON.info.version;
 
-    return callback(null, {
-      result: true,
-      output: [{
-        type: 'collection',
-        data: collectionJSON
-      }]
+      return callback(null, {
+        result: true,
+        output: [{
+          type: 'collection',
+          data: collectionJSON
+        }]
+      });
     });
   }
 
diff --git a/lib/utils.js b/lib/utils.js
@@ -22,6 +22,7 @@ module.exports = {
 
         // check the type of the value of that option came from the user
         switch (defaultOptions[id].type) {
+          case 'string':
           case 'boolean':
             if (typeof userOptions[id] === defaultOptions[id].type) {
               retVal[id] = userOptions[id];
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -121,7 +121,7 @@
     "commander": "2.20.3",
     "js-yaml": "3.13.1",
     "lodash": "4.17.20",
-    "oas-resolver-browser": "2.3.3",
+    "oas-resolver-browser": "2.5.0",
     "path-browserify": "1.0.1",
     "postman-collection": "3.6.6",
     "yaml": "1.8.3"
diff --git a/scripts/test-unit.sh b/scripts/test-unit.sh
@@ -38,6 +38,6 @@ fi
 
 # run test
 node --max-old-space-size=2048 ./node_modules/.bin/nyc ${COVERAGE_REPORT} --report-dir ./.coverage \
-  -x **/assets/** --print both ./node_modules/.bin/_mocha \
+  -x **/assets/** --print both ./node_modules/.bin/_mocha --timeout 15000 \
   --reporter ${MOCHA_REPORTER} --reporter-options output=${XUNIT_FILE} \
   test/unit/*.test.js --recursive --prof --grep "$1";
diff --git a/test/data/valid_openapi/remote-refs.yaml b/test/data/valid_openapi/remote-refs.yaml
@@ -0,0 +1,29 @@
+openapi: "3.0.0"
+info:
+  version: 1.0.0
+  title: Swagger Petstore
+  license:
+    name: MIT
+servers:
+  - url: http://petstore.swagger.io/v1
+paths:
+  /pets:
+    get:
+      summary: List all pets
+      operationId: listPets
+      parameters:
+        - $ref: 'https://raw.githubusercontent.com/postmanlabs/openapi-to-postman/develop/test/data/petstore%20separate%20yaml/spec/parameters.yaml#/tagsParam'
+        - $ref: 'https://raw.githubusercontent.com/postmanlabs/openapi-to-postman/develop/test/data/petstore%20separate%20yaml/spec/parameters.yaml#/limitsParam'
+      responses:
+        '200':
+          description: An paged array of pets
+          content:
+            application/json:    
+              schema:
+                $ref: "./spec/Pet.yaml"
+        default:
+          description: unexpected error
+          content:
+            application/json:
+              schema:
+                $ref: "./common/Error.yaml"
diff --git a/test/system/structure.test.js b/test/system/structure.test.js
@@ -8,6 +8,8 @@ const optionIds = [
     'folderStrategy',
     'indentCharacter',
     'requestNameSource',
+    'resolveRemoteRefs',
+    'sourceUrl',
     'includeAuthInfoInExample',
     'shortValidationErrors',
     'validationPropertiesToIgnore',
@@ -78,6 +80,18 @@ const optionIds = [
       ' If “Fallback” is selected, the request will be named after one of the following schema' +
       ' values: `description`, `operationid`, `url`.'
     },
+    resolveRemoteRefs: {
+      name: 'Resolve remote references',
+      type: 'boolean',
+      default: false,
+      description: 'Select whether to resolve remote references.'
+    },
+    sourceUrl: {
+      name: 'Source URL of definition',
+      type: 'string',
+      default: '',
+      description: 'Specify source URL of definition to resolve remote references mentioned in it.'
+    },
     includeAuthInfoInExample: {
       name: 'Include auth info in example requests',
       type: 'boolean',
diff --git a/test/unit/base.test.js b/test/unit/base.test.js
diff --git a/test/unit/sanity.test.js b/test/unit/sanity.test.js
diff --git a/test/unit/validator.test.js b/test/unit/validator.test.js
