index.js

    while (Date.now() < options.endTime && !account.twoFACode) {
      await sleep(options.heartBeat);
      account = await cozy.data.find('io.cozy.accounts', this.accountId);
      log('debug', `current accountState : ${account.state}`);
      log('debug', `current twoFACode : ${account.twoFACode}`);
    }

    if (account.twoFACode) {
      await this.resetTwoFAState();
      return account.twoFACode;
    }

    throw new Error('USER_ACTION_NEEDED.TWOFA_EXPIRED');
  }
  /**
   * Tells Cozy-Home that we have successfully logged in.
   * Useful when auto-success has been deactivated.
   * See `deactivateAutoSuccess`
   */


  async notifySuccessfulLogin() {
    log('debug', 'Notify Cozy-Home of successful login');
    await this.updateAccountAttributes({
      state: 'LOGIN_SUCCESS'
    });
  }
  /**
   * By default, cozy-home considers that the konnector has successfully logged in
   * when the konnector has run for more than 8s. This is problematic for 2FA since
   * the konnector can sit idle, just waiting for the 2FA to come back.
   *
   * When this method is called, cozy-home is notified and will not consider the
   * absence of error after 8s to be a success. Afterwards, to notify cozy-home when
   * the user has logged in successfully, for example, after the user has entered 2FA
   * codes, it is necessary to call `notifySuccessfulLogin`.
   *
   * Does nothing if called more than once.
   */


  async deactivateAutoSuccessfulLogin() {
    log('debug', 'Deactivating auto success for Cozy-Home');
    await this.updateAccountAttributes({
      state: 'HANDLE_LOGIN_SUCCESS'
    });
  }
  /**
   * This is saveBills function from cozy-konnector-libs which automatically adds sourceAccount in
   * metadata of each entry
   *
   * @returns {Promise} resolves with entries hydrated with db data
   */


  saveBills(entries, fields, options) {
    return saveBills(entries, fields, {
      sourceAccount: this.accountId,
      sourceAccountIdentifier: fields.login,
      ...options
    });
  }
  /**
   * This is saveFiles function from cozy-konnector-libs which automatically adds sourceAccount and
   * sourceAccountIdentifier cozyMetadatas to files
   *
   * @returns {Promise} resolves with the list of entries with file objects
   */


  saveFiles(entries, fields, options) {
    return saveFiles(entries, fields, {
      sourceAccount: this.accountId,
      sourceAccountIdentifier: fields.login,
      ...options
    });
  }
  /**
   * This is updateOrCreate function from cozy-konnector-libs which automatically adds sourceAccount in
   * metadata of each entry
   *
   * @returns {Promise} resolves to an array of db objects
   */


  updateOrCreate(entries, doctype, matchingAttributes, options) {
    return updateOrCreate(entries, doctype, matchingAttributes, {
      sourceAccount: this.accountId,
      sourceAccountIdentifier: get(options, 'fields.login'),
      ...options
    });
  }
  /**
   * This is saveIdentity function from cozy-konnector-libs which automatically adds sourceAccount in
   * metadata of each entry
   *
   * @returns {Promise} empty promise
   */


  saveIdentity(contact, accountIdentifier, options = {}) {
    return saveIdentity(contact, accountIdentifier, {
      sourceAccount: this.accountId,
      sourceAccountIdentifier: accountIdentifier,
      ...options
    });
  }
  /**
   * This is signin function from cozy-konnector-libs which automatically adds deactivateAutoSuccessfulLogin
   * and notifySuccessfulLogin calls
   *
   * @returns {Promise} resolve with an object containing form data
   */


  async signin(options = {}) {
    await this.deactivateAutoSuccessfulLogin();
    const result = await signin(omit(options, 'notifySuccessfulLogin'));

    if (options.notifySuccessfulLogin !== false) {
      await this.notifySuccessfulLogin();
    }

    return result;
  }
  /**
   * Send a special error code which is interpreted by the cozy stack to terminate the execution of the
   * connector now
   *
   * @param  {string} err - The error code to be saved as connector result see [docs/ERROR_CODES.md]
   *
   * @example
   * ```javascript
   * this.terminate('LOGIN_FAILED')
   * ```
   */


  terminate(err) {
    log('critical', String(err).substr(0, LOG_ERROR_MSG_LIMIT));
    captureExceptionAndDie(err);
  }
  /**
   * Get cozyMetaData from the context of the connector
   *
   * @param  {object} data - this data will be merged with cozyMetaData
   */


  getCozyMetadata(data) {
    Object.assign(data, {
      sourceAccount: this.accountId
    });
    return manifest.getCozyMetadata(data);
  }

}

wrapIfSentrySetUp(BaseKonnector.prototype, 'run');
BaseKonnector.findFolderPath = findFolderPath;
BaseKonnector.checkTOS = checkTOS;
module.exports = BaseKonnector;

/***/ }),
/* 1649 */
/***/ (function(module, exports, __webpack_require__) {

/**
 * Encapsulates the saving of Bills : saves the files, saves the new data, and associate the files
 * to an existing bank operation
 *
 * @module saveBills
 */
const utils = __webpack_require__(538);

const saveFiles = __webpack_require__(1650);

const hydrateAndFilter = __webpack_require__(376);

const addData = __webpack_require__(1670);

const log = __webpack_require__(2).namespace('saveBills');

const linkBankOperations = __webpack_require__(1671);

const DOCTYPE = 'io.cozy.bills';

const _ = __webpack_require__(1099);

const manifest = __webpack_require__(1093);

const requiredAttributes = {
  date: 'isDate',
  amount: 'isNumber',
  vendor: 'isString'
};
/**
 * Combines the features of `saveFiles`, `hydrateAndFilter`, `addData` and `linkBankOperations` for a
 * common case: bills.
 * Will create `io.cozy.bills` objects. The default deduplication keys are `['date', 'amount', 'vendor']`.
 * You need the full permission on `io.cozy.bills`, full permission on `io.cozy.files` and also
 * full permission on `io.cozy.bank.operations` in your manifest, to be able to use this function.
 *
 * Parameters:
 *
 * - `documents` is an array of objects with any attributes with some mandatory attributes :
 *   + `amount` (Number): the amount of the bill used to match bank operations
 *   + `date` (Date): the date of the bill also used to match bank operations
 *   + `vendor` (String): the name of the vendor associated to the bill. Ex: 'trainline'
 *   + `currency` (String) default: EUR:  The ISO currency value (not mandatory since there is a
 *   default value.
 *   + `contractId` (String): Contract unique identicator used to deduplicate bills
 *   + `contractLabel`: (String) User label if define, must be used with contractId
 *   + `matchingCriterias` (Object): criterias that can be used by an external service to match bills
 *   with bank operations. If not specified but the 'banksTransactionRegExp' attribute is specified in the
 *   manifest of the connector, this value is automatically added to the bill
 *
 *   You can also pass attributes expected by `saveFiles` : fileurl, filename, requestOptions
 *   and more
 *
 *   Please take a look at [io.cozy.bills doctype documentation](https://github.com/cozy/cozy-doctypes/blob/master/docs/io.cozy.bills.md)
 * - `fields` (object) this is the first parameter given to BaseKonnector's constructor
 * - `options` is passed directly to `saveFiles`, `hydrateAndFilter`, `addData` and `linkBankOperations`.
 *
 * @example
 *
 * ```javascript
 * const { BaseKonnector, saveBills } = require('cozy-konnector-libs')
 *
 * module.exports = new BaseKonnector(function fetch (fields) {
 *   const documents = []
 *   // some code which fills documents
 *   return saveBills(documents, fields, {
 *     identifiers: ['vendor']
 *   })
 * })
 * ```
 *
 * @alias module:saveBills
 */

const saveBills = async (inputEntries, fields, inputOptions = {}) => {
  // Cloning input arguments since both entries and options are expected
  // to be modified by functions called inside saveBills.
  const entries = _.cloneDeepWith(inputEntries, value => {
    // do not try to clone streams https://github.com/konnectors/libs/issues/682
    if (value && value.readable) {
      return value;
    }

    return undefined;
  });

  const options = _.cloneDeep(inputOptions);

  if (!_.isArray(entries) || entries.length === 0) {
    log('warn', 'saveBills: no bills to save');
    return Promise.resolve();
  }

  if (!options.sourceAccount) {
    log('warn', 'There is no sourceAccount given to saveBills');
  }

  if (!options.sourceAccountIdentifier) {
    log('warn', 'There is no sourceAccountIdentifier given to saveBills');
  }

  if (typeof fields === 'string') {
    fields = {
      folderPath: fields
    };
  } // Deduplicate on this keys


  options.keys = options.keys || Object.keys(requiredAttributes);
  const originalEntries = entries;

  const defaultShouldUpdate = (entry, dbEntry) => entry.invoice !== dbEntry.invoice || !dbEntry.cozyMetadata || !dbEntry.matchingCriterias;

  if (!options.shouldUpdate) {
    options.shouldUpdate = defaultShouldUpdate;
  } else {
    const fn = options.shouldUpdate;

    options.shouldUpdate = (entry, dbEntry) => {
      return defaultShouldUpdate(entry, dbEntry) || fn(entry, dbEntry);
    };
  }

  let tempEntries;
  tempEntries = manageContractsData(entries, options);
  tempEntries = await saveFiles(tempEntries, fields, options);

  if (options.processPdf) {
    let moreEntries = [];

    for (let entry of tempEntries) {
      if (entry.fileDocument) {
        let pdfContent;

        try {
          pdfContent = await utils.getPdfText(entry.fileDocument._id); // allow to create more entries related to the same file

          const result = await options.processPdf(entry, pdfContent.text, pdfContent);
          if (result && result.length) moreEntries = [...moreEntries, ...result];
        } catch (err) {
          log('warn', `processPdf: Failed to read pdf content in ${_.get(entry, 'fileDocument.attributes.name')}`);
          log('warn', err.message);
          entry.__ignore = true;
        }
      }
    }

    if (moreEntries.length) tempEntries = [...tempEntries, ...moreEntries];
  } // try to get transaction regexp from the manifest


  let defaultTransactionRegexp = null;

  if (Object.keys(manifest.data).length && manifest.data.banksTransactionRegExp) {
    defaultTransactionRegexp = manifest.data.banksTransactionRegExp;
  }

  tempEntries = tempEntries.filter(entry => !entry.__ignore) // we do not save bills without associated file anymore
  .filter(entry => entry.fileDocument).map(entry => {
    entry.currency = convertCurrency(entry.currency);
    entry.invoice = `io.cozy.files:${entry.fileDocument._id}`;
    const matchingCriterias = entry.matchingCriterias || {};

    if (defaultTransactionRegexp && !matchingCriterias.labelRegex) {
      matchingCriterias.labelRegex = defaultTransactionRegexp;
      entry.matchingCriterias = matchingCriterias;
    }

    delete entry.fileDocument;
    delete entry.fileAttributes;
    return entry;
  });
  checkRequiredAttributes(tempEntries);
  tempEntries = await hydrateAndFilter(tempEntries, DOCTYPE, options);
  tempEntries = await addData(tempEntries, DOCTYPE, options);

  if (options.linkBankOperations !== false) {
    tempEntries = await linkBankOperations(originalEntries, DOCTYPE, fields, options);
    log('debug', 'after linkbankoperation');
  }

  return tempEntries;
};

function convertCurrency(currency) {
  if (currency) {
    if (currency.includes('€')) {
      return 'EUR';
    } else if (currency.includes('$')) {
      return 'USD';
    } else if (currency.includes('£')) {
      return 'GBP';
    } else {
      return currency;
    }
  } else {
    return 'EUR';
  }
}

function checkRequiredAttributes(entries) {
  for (let entry of entries) {
    for (let attr in requiredAttributes) {
      if (entry[attr] == null) {
        throw new Error(`saveBills: an entry is missing the required ${attr} attribute`);
      }

      const checkFunction = requiredAttributes[attr];

      const isExpectedType = _(entry[attr])[checkFunction]();

      if (isExpectedType === false) {
        throw new Error(`saveBills: an entry has a ${attr} which does not respect ${checkFunction}`);
      }
    }
  }
}

function manageContractsData(tempEntries, options) {
  if (options.contractLabel && options.contractId === undefined) {
    log('warn', 'contractLabel used without contractId, ignoring it.');
    return tempEntries;
  }

  let newEntries = tempEntries; // if contractId passed by option

  if (options.contractId) {
    // Define contractlabel from contractId if not set in options
    if (!options.contractLabel) {
      options.contractLabel = options.contractId;
    } // Set saving path from contractLabel


    options.subPath = options.contractLabel; // Add contractId to deduplication keys

    addContractIdToDeduplication(options); // Add contract data to bills

    newEntries = newEntries.map(entry => addContractsDataToBill(entry, options)); // if contractId passed by bill attribute
  } else if (billsHaveContractId(newEntries)) {
    // Add contractId to deduplication keys
    addContractIdToDeduplication(options);
    newEntries = newEntries.map(entry => mergeContractsDataInBill(entry)); //manageContractsDataPassedByAttribute(newEntries, options
  }

  return newEntries;
}

function addContractsDataToBill(entry, options) {
  entry.contractLabel = options.contractLabel;
  entry.contractId = options.contractId;
  return entry;
}

function mergeContractsDataInBill(entry) {
  // Only treat bill with data
  if (entry.contractId) {
    // Surcharge label in needed
    if (!entry.contractLabel) {
      entry.contractLabel = entry.contractId;
    } // Edit subpath of each bill according to contractLabel


    entry.subPath = entry.contractLabel;
  }

  return entry;
}
/* This function return true if at least one bill of entries has a contractId
 */


function billsHaveContractId(entries) {
  for (const entry of entries) {
    if (entry.contractId) {
      return true;
    }
  }

  return false;
}
/* Add contractId to deduplication keys
 */


function addContractIdToDeduplication(options) {
  if (options.keys) {
    options.keys.push('contractId');
  }
}

module.exports = saveBills;
module.exports.manageContractsData = manageContractsData;

/***/ }),
/* 1650 */
/***/ (function(module, exports, __webpack_require__) {

/**
 * Saves the given files in the given folder via the Cozy API.
 *
 * @module saveFiles
 */
const bluebird = __webpack_require__(377);

const retry = __webpack_require__(1651);

const mimetypes = __webpack_require__(1095);

const path = __webpack_require__(159);

const requestFactory = __webpack_require__(22);

const omit = __webpack_require__(902);

const get = __webpack_require__(414);

const log = __webpack_require__(2).namespace('saveFiles');

const manifest = __webpack_require__(1093);

const cozy = __webpack_require__(539);

const {
  queryAll
} = __webpack_require__(538);

const mkdirp = __webpack_require__(1653);

const errors = __webpack_require__(1654);

const stream = __webpack_require__(99);

const fileType = __webpack_require__(1655);

const ms = 1;
const s = 1000 * ms;
const m = 60 * s;
const DEFAULT_TIMEOUT = Date.now() + 4 * m; // 4 minutes by default since the stack allows 5 minutes

const DEFAULT_CONCURRENCY = 1;
const DEFAULT_RETRY = 1; // do not retry by default

/**
 * Saves the files given in the fileurl attribute of each entries
 *
 * You need the full permission on `io.cozy.files` in your manifest to use this function.
 *
 * @param {Array} entries - list of object describing files to save
 * @param {string} entries.fileurl - The url of the file (can be a function returning the value). Ignored if `filestream` is given
 * @param {Function} entries.fetchFile - the connector can give it's own function to fetch the file from the website, which will be run only when necessary (if the corresponding file is missing on the cozy) function returning the stream). This function must return a promise resolved as a stream
 * @param {object} entries.filestream - the stream which will be directly passed to cozyClient.files.create (can also be function returning the stream)
 * @param {object} entries.requestOptions - The options passed to request to fetch fileurl (can be a function returning the value)
 * @param {string} entries.filename - The file name of the item written on disk. This attribute is optional and as default value, the file name will be "smartly" guessed by the function. Use this attribute if the guess is not smart enough for you, or if you use `filestream` (can be a function returning the value).
 * @param {string} entries.shouldReplaceName - used to migrate filename. If saveFiles finds a file linked to this entry and this file name matches `shouldReplaceName`, the file is renamed to `filename` (can be a function returning the value)
 * @param {Function} entries.shouldReplaceFile - use this function to state if the current entry should be forced to be redownloaded and replaced. Usefull if we know the file content can change and we always want the last version.
 * @param {object} entries.fileAttributes - ex: `{created_at: new Date()}` sets some additionnal file attributes passed to cozyClient.file.create
 * @param {string} entries.subPath - A subpath to save all files, will be created if needed.
 * @param {object} fields - is the argument given to the main function of your connector by the BaseKonnector.  It especially contains a `folderPath` which is the string path configured by the user in collect/home
 * @param {object} options - global options
 * @param {number} options.timeout - timestamp which can be used if your connector needs to fetch a lot of files and if the stack does not give enough time to your connector to fetch it all. It could happen that the connector is stopped right in the middle of the download of the file and the file will be broken. With the `timeout` option, the `saveFiles` function will check if the timeout has passed right after downloading each file and then will be sure to be stopped cleanly if the timeout is not too long. And since it is really fast to check that a file has already been downloaded, on the next run of the connector, it will be able to download some more files, and so on. If you want the timeout to be in 10s, do `Date.now() + 10*1000`.  You can try it in the previous code.
 * @param {number|boolean} options.contentType - ex: 'application/pdf' used to force the contentType of documents when they are badly recognized by cozy. If "true" the content type will be recognized from the file name and forced the same way.
 * @param {number} options.concurrency - default: `1` sets the maximum number of concurrent downloads
 * @param {Function} options.validateFile - default: do not validate if file is empty or has bad mime type
 * @param {boolean|Function} options.validateFileContent - default false. Also check the content of the file to recognize the mime type
 * @param {Array} options.fileIdAttributes - array of strings : Describes which attributes of files will be taken as primary key for files to check if they already exist, even if they are moved. If not given, the file path will used for deduplication as before.
 * @param {string} options.subPath - A subpath to save this file, will be created if needed.
 * @param {Function} options.fetchFile - the connector can give it's own function to fetch the file from the website, which will be run only when necessary (if the corresponding file is missing on the cozy) function returning the stream). This function must return a promise resolved as a stream
 *
 * @example
 * ```javascript
 * await saveFiles([{fileurl: 'https://...', filename: 'bill1.pdf'}], fields, {
 *    fileIdAttributes: ['fileurl']
 * })
 * ```
 *
 * @alias module:saveFiles
 */

const saveFiles = async (entries, fields, options = {}) => {
  if (!entries || entries.length === 0) {
    log('warn', 'No file to download');
  }

  if (!options.sourceAccount) {
    log('warn', 'There is no sourceAccount given to saveFiles');
  }

  if (!options.sourceAccountIdentifier) {
    log('warn', 'There is no sourceAccountIdentifier given to saveFIles');
  }

  if (typeof fields !== 'object') {
    log('debug', 'Deprecation warning, saveFiles 2nd argument should not be a string');
    fields = {
      folderPath: fields
    };
  }

  const saveOptions = {
    folderPath: fields.folderPath,
    fileIdAttributes: options.fileIdAttributes,
    timeout: options.timeout || DEFAULT_TIMEOUT,
    concurrency: options.concurrency || DEFAULT_CONCURRENCY,
    retry: options.retry || DEFAULT_RETRY,
    postProcess: options.postProcess,
    postProcessFile: options.postProcessFile,
    contentType: options.contentType,
    requestInstance: options.requestInstance,
    shouldReplaceFile: options.shouldReplaceFile,
    validateFile: options.validateFile || defaultValidateFile,
    subPath: options.subPath,
    sourceAccountOptions: {
      sourceAccount: options.sourceAccount,
      sourceAccountIdentifier: options.sourceAccountIdentifier
    }
  };

  if (options.validateFileContent) {
    if (options.validateFileContent === true) {
      saveOptions.validateFileContent = defaultValidateFileContent;
    } else if (typeof options.validateFileContent === 'function') {
      saveOptions.validateFileContent = options.validateFileContent;
    }
  }

  noMetadataDeduplicationWarning(saveOptions);

  const canBeSaved = entry => entry.fetchFile || entry.fileurl || entry.requestOptions || entry.filestream;

  let filesArray = undefined;
  let savedFiles = 0;
  const savedEntries = [];

  try {
    await bluebird.map(entries, async entry => {
      ;
      ['fileurl', 'filename', 'shouldReplaceName', 'requestOptions' // 'filestream'
      ].forEach(key => {
        if (entry[key]) entry[key] = getValOrFnResult(entry[key], entry, options);
      });

      if (entry.filestream && !entry.filename) {
        log('warn', 'Missing filename property for for filestream entry, entry is ignored');
        return;
      }

      if (entry.shouldReplaceName) {
        // At first encounter of a rename, we set the filenamesList
        if (filesArray === undefined) {
          log('debug', 'initialize files list for renamming');
          filesArray = await getFiles(fields.folderPath);
        }

        const fileFound = filesArray.find(f => getAttribute(f, 'name') === entry.shouldReplaceName);

        if (fileFound) {
          await renameFile(fileFound, entry); // we continue because saveFile mays also add fileIdAttributes to the renamed file
        }

        delete entry.shouldReplaceName;
      }

      if (canBeSaved(entry)) {
        const folderPath = await getOrCreateDestinationPath(entry, saveOptions);
        entry = await saveEntry(entry, { ...saveOptions,
          folderPath
        });

        if (entry && entry._cozy_file_to_create) {
          savedFiles++;
          delete entry._cozy_file_to_create;
        }
      }

      savedEntries.push(entry);
    }, {
      concurrency: saveOptions.concurrency
    });
  } catch (err) {
    if (err.message !== 'TIMEOUT') {
      throw err;
    } else {
      log('warn', `saveFile timeout: still ${entries.length - savedEntries.length} / ${entries.length} to download`);
    }
  }

  log('info', `saveFiles created ${savedFiles} files for ${savedEntries ? savedEntries.length : 'n'} entries`);
  return savedEntries;
};

const saveEntry = async function (entry, options) {
  if (options.timeout && Date.now() > options.timeout) {
    const remainingTime = Math.floor((options.timeout - Date.now()) / s);
    log('info', `${remainingTime}s timeout finished for ${options.folderPath}`);
    throw new Error('TIMEOUT');
  }

  let file = await getFileIfExists(entry, options);
  let shouldReplace = false;

  if (file) {
    try {
      shouldReplace = await shouldReplaceFile(file, entry, options);
    } catch (err) {
      log('info', `Error in shouldReplace : ${err.message}`);
      shouldReplace = true;
    }
  }

  let method = 'create';

  if (shouldReplace && file) {
    method = 'updateById';
    log('debug', `Will replace ${getFilePath({
      options,
      file
    })}...`);
  }

  try {
    if (!file || method === 'updateById') {
      log('debug', omit(entry, 'filestream'));
      logFileStream(entry.filestream);
      log('debug', `File ${getFilePath({
        options,
        entry
      })} does not exist yet or is not valid`);
      entry._cozy_file_to_create = true;
      file = await retry(createFile, {
        interval: 1000,
        throw_original: true,
        max_tries: options.retry,
        args: [entry, options, method, file ? file._id : undefined]
      }).catch(err => {
        if (err.message === 'BAD_DOWNLOADED_FILE') {
          log('warn', `Could not download file after ${options.retry} tries removing the file`);
        } else {
          log('warn', 'unknown file download error: ' + err.message);
        }
      });
    }

    attachFileToEntry(entry, file);
    sanitizeEntry(entry);

    if (options.postProcess) {
      await options.postProcess(entry);
    }
  } catch (err) {
    if (getErrorStatus(err) === 413) {
      // the cozy quota is full
      throw new Error(errors.DISK_QUOTA_EXCEEDED);
    }

    log('warn', errors.SAVE_FILE_FAILED);
    log('warn', err.message, `Error caught while trying to save the file ${entry.fileurl ? entry.fileurl : entry.filename}`);
  }

  return entry;
};

function noMetadataDeduplicationWarning(options) {
  const fileIdAttributes = options.fileIdAttributes;

  if (!fileIdAttributes) {
    log('warn', `saveFiles: no deduplication key is defined, file deduplication will be based on file path`);
  }

  const slug = manifest.data.slug;

  if (!slug) {
    log('warn', `saveFiles: no slug is defined for the current connector, file deduplication will be based on file path`);
  }

  const sourceAccountIdentifier = get(options, 'sourceAccountOptions.sourceAccountIdentifier');

  if (!sourceAccountIdentifier) {
    log('warn', `saveFiles: no sourceAccountIdentifier is defined in options, file deduplication will be based on file path`);
  }
}

async function getFileIfExists(entry, options) {
  const fileIdAttributes = options.fileIdAttributes;
  const slug = manifest.data.slug;
  const sourceAccountIdentifier = get(options, 'sourceAccountOptions.sourceAccountIdentifier');
  const isReadyForFileMetadata = fileIdAttributes && slug && sourceAccountIdentifier;

  if (isReadyForFileMetadata) {
    const file = await getFileFromMetaData(entry, fileIdAttributes, sourceAccountIdentifier, slug);

    if (!file) {
      // no file with correct metadata, maybe the corresponding file already exist in the default
      // path from a previous version of the connector
      return await getFileFromPath(entry, options);
    } else return file;
  } else {
    return await getFileFromPath(entry, options);
  }
}

async function getFileFromMetaData(entry, fileIdAttributes, sourceAccountIdentifier, slug) {
  const index = await cozy.data.defineIndex('io.cozy.files', ['metadata.fileIdAttributes', 'trashed', 'cozyMetadata.sourceAccountIdentifier', 'cozyMetadata.createdByApp']);
  log('debug', `Checking existence of ${calculateFileKey(entry, fileIdAttributes)}`);
  const files = await queryAll('io.cozy.files', {
    metadata: {
      fileIdAttributes: calculateFileKey(entry, fileIdAttributes)
    },
    trashed: false,
    cozyMetadata: {
      sourceAccountIdentifier,
      createdByApp: slug
    }
  }, index);

  if (files && files[0]) {
    if (files.length > 1) {
      log('warn', `Found ${files.length} files corresponding to ${calculateFileKey(entry, fileIdAttributes)}`);
    }

    return files[0];
  } else {
    log('debug', 'not found');
    return false;
  }
}

async function getFileFromPath(entry, options) {
  try {
    log('debug', `Checking existence of ${getFilePath({
      entry,
      options
    })}`);
    const result = await cozy.files.statByPath(getFilePath({
      entry,
      options
    }));
    return result;
  } catch (err) {
    log('debug', err.message);
    return false;
  }
}

async function createFile(entry, options, method, fileId) {
  const folder = await cozy.files.statByPath(options.folderPath);
  let createFileOptions = {
    name: getFileName(entry),
    dirID: folder._id
  };

  if (options.contentType) {
    if (options.contentType === true && entry.filename) {
      createFileOptions.contentType = mimetypes.contentType(entry.filename);
    } else {
      createFileOptions.contentType = options.contentType;
    }
  }

  createFileOptions = { ...createFileOptions,
    ...entry.fileAttributes,
    ...options.sourceAccountOptions
  };

  if (options.fileIdAttributes) {
    createFileOptions = { ...createFileOptions,
      ...{
        metadata: { ...createFileOptions.metadata,
          fileIdAttributes: calculateFileKey(entry, options.fileIdAttributes)
        }
      }
    };
  }

  let toCreate;

  if (entry.filestream) {
    toCreate = entry.filestream;
  } else if (entry.fetchFile || options.fetchFile) {
    toCreate = await (entry.fetchFile || options.fetchFile)(entry);
  } else {
    toCreate = downloadEntry(entry, { ...options,
      simple: false
    });
  }

  let fileDocument;

  if (method === 'create') {
    fileDocument = await cozy.files.create(toCreate, createFileOptions);
  } else if (method === 'updateById') {
    log('debug', `replacing file for ${entry.filename}`);
    fileDocument = await cozy.files.updateById(fileId, toCreate, createFileOptions);
  }

  if (options.validateFile) {
    if ((await options.validateFile(fileDocument)) === false) {
      await removeFile(fileDocument);
      throw new Error('BAD_DOWNLOADED_FILE');
    }

    if (options.validateFileContent && !(await options.validateFileContent(fileDocument))) {
      await removeFile(fileDocument);
      throw new Error('BAD_DOWNLOADED_FILE');
    }
  }

  return fileDocument;
}

function downloadEntry(entry, options) {
  let filePromise = getRequestInstance(entry, options)(getRequestOptions(entry, options));

  if (options.contentType) {
    // the developper wants to force the contentType of the document
    // we pipe the stream to remove headers with bad contentType from the request
    return filePromise.pipe(new stream.PassThrough());
  } // we have to do this since the result of filePromise is not a stream and cannot be taken by
  // cozy.files.create


  if (options.postProcessFile) {
    log('warn', 'Be carefull postProcessFile option is deprecated. You should use the filestream attribute in each entry instead');
    return filePromise.then(data => options.postProcessFile(data));
  }

  filePromise.catch(err => {
    log('warn', `File download error ${err.message}`);
  });
  return filePromise;
}

const shouldReplaceFile = async function (file, entry, options) {
  const isValid = !options.validateFile || (await options.validateFile(file));

  if (!isValid) {
    log('warn', `${getFileName({
      file,
      options
    })} is invalid`);
    throw new Error('BAD_DOWNLOADED_FILE');
  }

  const defaultShouldReplaceFile = (file, entry) => {
    // replace all files with meta if there is file metadata to add
    const fileHasNoMetadata = !getAttribute(file, 'metadata');
    const fileHasNoId = !getAttribute(file, 'metadata.fileIdAttributes');
    const entryHasMetadata = !!get(entry, 'fileAttributes.metadata');
    const hasSourceAccountIdentifierOption = !!get(options, 'sourceAccountOptions.sourceAccountIdentifier');
    const fileHasSourceAccountIdentifier = !!getAttribute(file, 'cozyMetadata.sourceAccountIdentifier');
    const result = fileHasNoMetadata && entryHasMetadata || fileHasNoId && !!options.fileIdAttributes || hasSourceAccountIdentifierOption && !fileHasSourceAccountIdentifier;
    return result;
  };

  const shouldReplaceFileFn = entry.shouldReplaceFile || options.shouldReplaceFile || defaultShouldReplaceFile;
  return shouldReplaceFileFn(file, entry);
};

const removeFile = async function (file) {
  await cozy.files.trashById(file._id);
  await cozy.files.destroyById(file._id);
};

module.exports = saveFiles;
module.exports.getFileIfExists = getFileIfExists;

function getFileName(entry) {
  let filename;

  if (entry.filename) {
    filename = entry.filename;
  } else if (entry.fileurl) {
    // try to get the file name from the url
    const parsed = __webpack_require__(82).parse(entry.fileurl);

    filename = path.basename(parsed.pathname);
  } else {
    log('error', 'Could not get a file name for the entry');
    return false;
  }

  return sanitizeFileName(filename);
}

function sanitizeFileName(filename) {
  return filename.replace(/^\.+$/, '').replace(/[/?<>\\:*|":]/g, '');
}

function checkFileSize(fileobject) {
  const size = getAttribute(fileobject, 'size');
  const name = getAttribute(fileobject, 'name');

  if (size === 0 || size === '0') {
    log('warn', `${name} is empty`);
    log('warn', 'BAD_FILE_SIZE');
    return false;
  }

  return true;
}

function checkMimeWithPath(fileDocument) {
  const mime = getAttribute(fileDocument, 'mime');
  const name = getAttribute(fileDocument, 'name');
  const extension = path.extname(name).substr(1);

  if (extension && mime && mimetypes.lookup(extension) !== mime) {
    log('warn', `${name} and ${mime} do not correspond`);
    log('warn', 'BAD_MIME_TYPE');
    return false;
  }

  return true;
}

function logFileStream(fileStream) {
  if (!fileStream) return;

  if (fileStream && fileStream.constructor && fileStream.constructor.name) {
    log('debug', `The fileStream attribute is an instance of ${fileStream.constructor.name}`);
  } else {
    log('debug', `The fileStream attribute is a ${typeof fileStream}`);
  }
}

async function getFiles(folderPath) {
  const dir = await cozy.files.statByPath(folderPath);
  const files = await queryAll('io.cozy.files', {
    dir_id: dir._id
  });
  return files;
}

async function renameFile(file, entry) {