public async getNewsContent(newsURL: string): Promise<NewsContent> {
    try {
      await delay(5000);
      const response = await request({
        uri: newsURL,
        headers: commonHeaders,
        encoding: null,
      });

      let $ = cheerio.load(response);
      $('script').remove();
      const encodedResponse = iconv.decode(response, $('meta[charset]').attr('charset')).toString();
      $ = cheerio.load(encodedResponse);

      const newsTitle = $('meta[property=\'og:title\']').attr('content');
      const newsContent = $('#articeBody, #newsEndContents, #articleBodyContents');

      return {
        title: newsTitle,
        content: newsContent.html(),
        plain_text: newsContent.text(),
        press: $('.press_logo img, #pressLogo img').attr('alt'),
        link: newsURL,
      };
    } catch (err) {
      logger.error(err);
    }
  }

export async function parseSeiten( url: string | URL ) {
  const html = await request( url );

  const results: Seite[] = [];

  const $ = load( html );

  if ( $( '.adbbody table' ).length ) {
    results.push( {
      selektion: 'Alle',
      institutionen: parseInstitutionen( $ )
    } );
  } else if ( $( '.adbbody ul.adbul1 li' ).length ) {
    const selektion = parseSelektion( $ );
    for ( let { url, text } of selektion ) {
      const html = await request( baseURL( url ) );
      const $ = load( html );
      results.push( {
        selektion: text,
        institutionen: parseInstitutionen( $ )
      } );
    }
  }

  return results;
}

  public async get(): Promise<void> {
    const baseUrl = 'https://www.jw.org';
    const url = `${baseUrl}/cmn-hans/出版物/音乐/高声欢唱/`;
    const urlT = `${baseUrl}/cmn-hant/出版物/音樂/高聲歡唱/`;

    const downloader = new GenericDownloader();

    const response = await downloader.download(url);
    const responseT = await downloader.download(urlT);

    const $list = cheerio.load(response);
    const $listT = cheerio.load(responseT);

    const xpath = '.musicList .musicFileContainer .fileTitle a';

    const links = $list(xpath).toArray();
    const linksT = $listT(xpath).toArray();

    let song = 0;

    const parser = new Parser();
    let i = 0;
    for (const link of links) {
      song++;

      const songLink = baseUrl + $list(link).attr('href');
      const songLinkT = baseUrl + $listT(linksT[i]).attr('href');

      const response = await downloader.download(songLink);
      const responseT = await downloader.download(songLinkT);

      const $ = cheerio.load(response);
      const $T = cheerio.load(responseT);

      const dirname = `${__dirname}/../../../../../../../ui/public/static/songs/`;

      const parsedDownload = await parser.parse($);

      await writeFile(
        `${dirname}cmn-hans/${song}.json`,
        JSON.stringify({ lines: parsedDownload.text }),
      );

      const parsedDownloadT = await parser.parse($T, undefined, $);

      await writeFile(
        `${dirname}cmn-hant/${song}.json`,
        JSON.stringify({ lines: parsedDownloadT.text }),
      );

      i++;
    }
  }

 /**
  * Transforme le bout de page passé en paramètre en objet classement
  * 
  * @param ligne html a parser.
  * @return objet classement
  */
 public static parseClassementLine(line /**: DOM Element */): ClassementLine {
     var $ = cheerio.load(line);
     var lineChildren = $(line).children();
     if (lineChildren !== null && lineChildren.length > 3) {
         
         var nom = $(lineChildren[1]).text().trim().toUpperCase(),
             points = $(lineChildren[2]).text(),
             joue = $(lineChildren[3]).text(),
             victoire = $(lineChildren[4]).text(),
             nul = $(lineChildren[5]).text(),
             defaite = $(lineChildren[6]).text(),
             bp = $(lineChildren[8]).text(),
             bc = $(lineChildren[9]).text(),
             diff = $(lineChildren[11]).text();
             
         var classementLine = new ClassementLine();
         classementLine.nom = nom;
         classementLine.points = points;
         classementLine.joue = joue;
         classementLine.gagne = victoire;
         classementLine.nul = nul;
         classementLine.perdu = defaite;
         classementLine.bp = bp;
         classementLine.bc = bc;
         classementLine.diff = diff;
         return classementLine;
     } else {
         return null;
     }
 }

     Utils.downloadData(url, function(result) { 
         // do parse
         var $5 = cheerio.load(result);
         var title = $5('.post .title').text().trim();
         var dateString = $5('.post .postmeta').text().trim();
         var contents = $5('.post .entry').children();
         var article="";
         for(var i=0; i< contents.length;i++) {
             if($5(contents[i]).attr('class') === 'sociable') {
                 break;
             }
             
             article += $5(contents[i]).text().trim();
             article += "\n";
         }
         
         var jour = dateString.split(' ')[0];
 
         if (jour.length === 1) {
             jour = '0' + jour;
         }
 
         var mois = listeMoisActu[dateString.split(' ')[1]],
             annee = dateString.split(' ')[2];
         
         var resultats = new Article();
         resultats.title = title;
         resultats.date = annee + '-' + mois + '-' + jour + ' ' + '00:00:00';
         resultats.article = article;
         callback(resultats);
     }, function(err) {

syncTest('200 ok with title', (test) => {
  let response = syncRequest('GET', settings.testSiteOrigin3);
  test.equal(response.status, 200);
  let $ = cheerio.load(response.body);
  //console.log("bd: " + response.body);
  test.equal($('body h1').text(), 'broken_test');
});

    export async function getUrlInfo(url: string) {
        if(!url.includes('http://') && !url.includes('https://')) {
            url = `http://${url}`;
        }

        const html = await request(url);
        const $ = cheerio.load(html);
        let htmlInfo: any = {
            'og:title':null,
            'og:description':null,
            'og:image':null
        };
        const meta = $('meta');
        const keys = Object.keys(meta);
        for (let s in htmlInfo) {
            keys.forEach(function(key) {
                if ( meta[key].attribs
                    && meta[key].attribs.property
                    && meta[key].attribs.property === s) {
                    htmlInfo[s] = meta[key].attribs.content;
                }
            })
        }
        htmlInfo.title = $('title').html();

        return htmlInfo;
    }

export const parseAccount = (html: string): Account => {
  const $ = cheerio.load(html);
  const result: { [key: string]: string } = {};

  $('#content #tab-5 .column tbody tr').each((rowIndex, tableRow) => {
    const cells: string[] = [];

    $(tableRow)
      .find('td')
      .each((cellIndex, tableCell) => {
        cells.push($(tableCell).html() || '');
      });

    result[camelCaseify(cells[0], ' ')] = parseScrapedText(cells[1]);
  });

  return {
    firstName: result.firstName,
    lastName: result.lastName,
    address: result.address,
    dateOfBirth: result.dateOfBirth,
    phoneNumber: result.phoneNumber,
    mobileNumber: result.mobileNumber,
    emailAddress: result.emailAddress,
    nameOnCard: result.nameOnCard,
    cardType: result.cardType,
    cardNumber: result.cardNumber,
    cardExpires: result.cardExpires,
    password: result.password,
    opalPin: result.opalPin,
    securityQuestion: result.securityQuestion,
    securityAnswer: result.securityAnswer,
  };
};

export const parseTransactions = (html: string): Transaction[] => {
  const $ = cheerio.load(html);
  const data: Transaction[] = [];

  const noData = $('.no-activity-list').text();
  if (noData !== '') {
    return [];
  }

  $('#transaction-data tbody tr').each((rowIndex, tableRow) => {
    const cells: string[] = [];

    $(tableRow)
      .find('td')
      .each((cellIndex, tableCell) => {
        cells.push($(tableCell).html() || '');
      });

    const [
      transaction,
      date,
      mode,
      details,
      journey,
      fareApplied,
      fare,
      discount,
      amount,
    ] = cells;

    const dataJson: Transaction = {
      transactionNumber: parseInt(transaction, 10),
      timestamp: parseTransactionDate(date),
      summary: sanitizeSummaryString(details) || null,
      mode: parseTransactionMode(mode) || null,
      fare: {
        applied: fareApplied || null,
        price: Math.abs(dollarToInt(fare)) || 0,
        discount: Math.abs(dollarToInt(discount)) || 0,
        paid: Math.abs(dollarToInt(amount)) || 0,
      },
      journey: null,
    };

    if (dataJson.mode && /^(ferry|bus|train)$/.test(dataJson.mode)) {
      const journeyData = dataJson.summary!.split(' to ');
      if (journeyData.length === 2) {
        dataJson.journey! = {
          number: parseInt(journey, 10),
          start: journeyData[0],
          end: journeyData[1],
        };
      }
    }

    data.push(dataJson);
  });

  return data;
};