test.js

'use strict';

require('mocha');
const assert = require('assert');
const request = require('./');

function next(limit) {
  return async function(url, res, acc) {
    const regex = /href=[^\s]*?\/page\/(\d+)\//g;
    let matches;
    while (matches = regex.exec(res.data)) {
      if (matches === null) {
        break;
      }
      const num = parseInt(matches[1], 10);
      if (isNaN(num)) {
        break;
      }
      const url = `${acc.orig}/page/${num}/`;
      // If we have already requested this `url`, we will skip it, and try the next `url` on the page.
      if (acc.urls.includes(url)) {
        continue;
      }
      if (num > limit) {
        break;
      }
      return url;
    }
  };
}

describe('paged-request', function() {
  this.timeout(10000);

  it('should make a request for paged content', function() {
    const limit = 2;
    return request('https://www.smashingmagazine.com/category/css', {}, next(limit))
      .then(acc => {
        assert.strictEqual(acc.pages.length, limit);
      });
  });

  it('should throw an error when page does not exist', function() {
    const limit = 1;
    return request('https://www.smashingmagazine.com/dflsjfslkfskfjds', {}, next(limit))
      .then(() => {
        throw new Error('expected an error');
      })
      .catch(err => {
        assert.ok(/404/.test(err.message));
      });
  });

  it('should keep the history of `urls`', function() {
    const limit = 3;
    return request('https://www.smashingmagazine.com/category/css', {}, next(limit))
      .then(acc => {
        assert.deepStrictEqual(acc.urls, [
          'https://www.smashingmagazine.com/category/css',
          'https://www.smashingmagazine.com/category/css/page/2/',
          'https://www.smashingmagazine.com/category/css/page/3/'
        ]);
      });
  });
});