This is Part 2 of a series. Refer to Part 1: How We Store NPM in GitHub and MongoDB, if needed. We begin Part 2 with MongoDB and AppFog already set up and running.

Now that we have a server crunching away to build the database, we need a way to query that data. We'll set up a separate Node.js server running hapi.js to serve the API.

ScoutJS Example Search

Packages

  1. We'll start by setting up an endpoint to get a single package's data.

    var creds = require('./credentials.js');
    var Joi = require('joi');
    var _ = require('lodash');
    var Q = require('q');
    
    var mongodbUrl = 'mongodb://' + creds.mongodbHost + ':27017/scout';
    var MongoClient = require('mongodb').MongoClient
    
    module.exports = {
      path: '/api/packages/{id}',
      method: 'GET',
      config: {
        validate: {
          params: {
            id: Joi.required(),
          }
        },
    
        handler: function (request, reply) {
          var id = request.params.id;
          var deferred = Q.defer();
    
          MongoClient.connect(mongodbUrl, function (err, db) {
            var collection = db.collection('packages');
    
            collection.findOne({"id":id}, function (err, record) {
              if (null != err) {
                deferred.reject(err);
              }
              else {
                deferred.resolve(record);
              }
              db.close();
            });
          })
          return deferred.promise
            .then(function (result) {
              reply(result);
            })
            .fail(function (err) {
              console.error('get failed', err);
              reply({ error: err }).code(500);
            });
        }
      }
    }
    
  2. Here we simply make a MongoDB findOne call and receive the JSON back to deliver to the client.

Search

  1. Next, we'll set up an endpoint to query the data. We'll pass the query in the URL and, optionally, the result offset for pagination.

    var FIELDS_TO_SEARCH = [
      'name',
      'description',
      'author',
      'keywords',
    ];
    
    var searchByQuery = {
      path: '/api/search/{query}/{offset?}',
      method: 'GET',
      config: {
        validate: {
          params: {
            query: Joi.string().required(),
            offset: Joi.number(),
          }
        },
    
        handler: function (request, reply) {
          var query = request.params.query;
          var offset = request.params.offset || 0;
          var deferred = Q.defer();
    
          var builtQuery = { '$or' : [] };
    
          // Build a MongoDB search query.
          FIELDS_TO_SEARCH.forEach(function (fieldName) {
            var queryObj = {};
    
            queryObj[fieldName] = { "$regex"   : query,
                                    "$options" : 'i' };
    
            builtQuery['$or'].push(queryObj);
          });
    
          // Execute the MongoDB search
          MongoClient.connect(mongodbUrl, function (err, db) {
            var collection = db.collection('packages');
    
            collection.find(builtQuery)
              .sort({ "name": 1 })
              .skip(offset)
              .toArray(function (err, docs) {
                if (null != err) {
                  deferred.reject(err);
                }
                else {
                  deferred.resolve(docs);
                }
    
                db.close();
              });
          });
    
          return deferred.promise
            .then(function (result) {
              reply(result);
            })
            .fail(function (err) {
              console.error('get failed', err);
              reply({ error: err }).code(500);
            });
        }
      }
    };
    
  2. We only want to search certain fields; so we set our query to specify the field name. Here's an example:

    value.name: (react) OR value.description: (react)
    
  3. We also want to clean up the search results because we only want to return a subset of the data for each result. Otherwise, our response would be huge.

    function cleanSearchResults (result) {
    var packages = _.get(result, 'body.results');
    var nextOffset = findSearchOffset(_.get(result, 'body.next'));
    
    var cleanPackages = _.map(packages, function (item, index) {
      var package = _.get(item, 'value');
    
      var output = _.pick(package, 'name', 'description', 'version');
    
      output.id = _.get(item, 'path.key');
      output.downloads = _.get(package, 'downloads.daily_total');
      output.avatar = _.get(package, 'github.owner.avatar_url');
      output.stars = _.get(package, 'github.stargazers_count');
      output.watchers = _.get(package, 'github.watchers_count');
      output.forks = _.get(package, 'github.forks_count');
      output.rank = _.get(package, 'npf_rank');
    
      output.created = _.get(package, 'created_date');
      output.updated = _.get(package, 'modified_date');
    
      return output;
    });
    
    return {
      packages: cleanPackages,
      next: nextOffset,
      results: _.get(result, 'body.total_count'),
    };
    }
    
    function findSearchOffset (nextLink) {
    if (!nextLink || !_.isString(nextLink)) return 0;
    var match = nextLink.match(/offset=(\d+)/i);
    if (!match || !_.isArray(match)) return 0;
    
    return parseInt(match[1]);
    };
    
  4. Here we loop over the results to process each package and return a new array of package objects. We'll use Lodash's .get() method in case some of the data isn't there. For example, say we want to get package.github.watcherscount. If there isn't a Github object in the data, our code would throw an error. Using Lo-Dash, we just get an undefined response instead.

    Note: Check out the Github repo for the full Hapi configuration: index.js

  5. Now that we have our /api/search endpoint and /api/packages endpoint, we can start making some test requests. For example, we can make a GET request to /api/search/react and get this JSON response (truncated for space):

    {
    "packages": [
      {
        "name": "react",
        "description": "React is a JavaScript library for building user interfaces.",
        "version": "0.13.3",
        "id": "react",
        "downloads": 465200,
        "avatar": "https://avatars.githubusercontent.com/u/69631?v=3",
        "stars": 26901,
        "watchers": 26901,
        "forks": 4011,
        "rank": 2.7451200000000004,
        "created": "2011-10-26T17:46:21.942Z",
        "updated": "2015-08-03T21:33:47.972Z"
      },
      ...
    ],
    "next": 10,
    "results": 3796
    }
    

Summary

With the API Server working we can start building the frontend to search and display all this wonderful data we have.