CrawlerOptions.java

Go to the documentation of this file.

/*
 * Turró i Cutiller Foundation. License notice.
 * Copyright (C) 2022 Lluis Turró Cutiller <http://www.turro.org/>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
package org.turro.elephant.servlet.crawler;
 
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.turro.elephant.context.ElephantContext;
import org.turro.file.Document;
import org.turro.http.activity.ActivityAction;
import org.turro.http.activity.IActivityOptions;
import org.turro.json.IJSONizable;
import org.turro.lock.Executable;
import org.turro.string.Strings;
 
public class CrawlerOptions implements IActivityOptions, IJSONizable {
 
  private boolean active;
  
  private int maxMinutes;
  private int maxRequests;
  
  private final Set<String> badRequest = new HashSet<>();
  private final Set<String> badAgent = new HashSet<>();
  private final Set<String> allowedDomains = new HashSet<>();
  
  private transient AllowedSet allowedSet;
  
  // To remove in future releases
  private Set<String> whiteList;
  private Set<String> blackList;
  
  @Override
  public boolean isActive() {
    return active;
  }
 
  @Override
  public int maxMinutes() {
    return maxMinutes;
  }
 
  @Override
  public int maxRequests() {
    return maxRequests;
  }
 
  @Override
  public boolean isWhiteListed(String ip) {
    return allowedSet != null && allowedSet.isWhiteListed(ip);
  }
 
  @Override
  public boolean isBlackListed(String ip) {
    return allowedSet != null && allowedSet.isBlackListed(ip);
  }
 
  @Override
  public boolean isBadRequest(String request) {
    return request == null || badRequest.stream().anyMatch(br -> request.matches(br));
  }
 
  @Override
  public boolean isBadAgent(String agent) {
    return agent == null || badAgent.stream().anyMatch(br -> agent.matches(br));
  }
 
  @Override
  public boolean isAllowedDomain(String domain) {
    return !Strings.isBlank(domain) && allowedDomains.stream().anyMatch(wd -> domain.endsWith(wd));
  }
 
  @Override
  public void whiteList(ActivityAction action) {
    CrawlerLists cl = loadLists();
    cl.getWhiteList().add(action);
    saveLists(cl);
    setAllowedFrom(cl);
  }
 
  @Override
  public void blackList(ActivityAction action) {
    CrawlerLists cl = loadLists();
    cl.getBlackList().add(action);
    saveLists(cl);
    setAllowedFrom(cl);
  }
 
  @Override
  public Consumer<ActivityAction> onWhiteList() {
    return (action) -> {
//      MailSenders.getPool()
//              .addAdministrators()
//              .send("Whitelisted", 
//                      Phrases.start("Whitelisted", action.getIp(), ":", action.getDomain())
//                              .line().add(action.getReason()).toString());
    };
  }
 
  @Override
  public Consumer<ActivityAction> onBlackList() {
    return (action) -> {
//      MailSenders.getPool()
//              .addAdministrators()
//              .send("Blacklisted", 
//                      Phrases.start("Blacklisted", action.getIp(), ":", action.getDomain())
//                              .line().add(action.getReason()).toString());
    };
  }
 
  /* Factory */
  
  public final static transient String CRAWLERS_RESPONSE = "/WEB-INF/elephant/activity/crawlers.html";
 
  public static CrawlerOptions instance() {
    return load();
  }
  
  /* Storage */
  
  private final static transient String 
          CRAWLERS_FILE = "/WEB-INF/elephant/activity/crawlers.json",
          CRAWLERS_LIST = "/WEB-INF/elephant/activity/crawlers-lists.json";
  
  private CrawlerLists loadLists() {
    Document crawlers = Document.from(ElephantContext.getRealPath(CRAWLERS_LIST));
    if(crawlers.exists()) {
      try {
        CrawlerLists cl = IJSONizable.fromJson(
                crawlers.content(), CrawlerLists.class);
        if(whiteList != null && blackList != null && (!whiteList.isEmpty() || !blackList.isEmpty())) { 
          whiteList.forEach(ip -> cl.getWhiteList().add(new ActivityAction(ip, ip, "")));
          blackList.forEach(ip -> cl.getBlackList().add(new ActivityAction(ip, ip, "")));
          whiteList = null;
          blackList = null;
          saveDefinitions();
          saveLists(cl);
        }
        return cl;
      } catch (Exception ex) {
        crawlers.delete();
        Logger.getLogger(CrawlerOptions.class.getName()).log(Level.SEVERE, null, ex);
      }
    }
    return new CrawlerLists();
  }
  
  private void setAllowedFrom(CrawlerLists cl) {
    allowedSet = cl.getAllowedSet();
  }
 
  private void saveDefinitions() {
    save(this, CRAWLERS_FILE);
  }
  
  private void saveLists(CrawlerLists cl) {
    save(cl, CRAWLERS_LIST);
  }
  
  private static final Executable LOCK = new Executable();
  
  private void save(IJSONizable jSONizable, String file) {
    LOCK.run(() -> {
      Document crawlers = Document.from(ElephantContext.getRealPath(file));
      try {
        crawlers.content(jSONizable.toJson());
      } catch (IOException ex) {
        Logger.getLogger(CrawlerOptions.class.getName()).log(Level.SEVERE, null, ex);
      }
    });
  }
  
  private static CrawlerOptions load() {
    Document crawlers = Document.from(ElephantContext.getRealPath(CRAWLERS_FILE));
    if(crawlers.exists()) {
      try {
        CrawlerOptions co = IJSONizable.fromJson(
                crawlers.content(), CrawlerOptions.class);
        addDefaults(co);
        co.setAllowedFrom(co.loadLists());
        return co;
      } catch (Exception ex) {
        crawlers.delete();
        Logger.getLogger(CrawlerOptions.class.getName()).log(Level.SEVERE, null, ex);
      }
    }
    return createDefaults();
  }
  
  private static CrawlerOptions createDefaults() {
    CrawlerOptions co = new CrawlerOptions();
    co.active = true;
    co.maxMinutes = 15;
    co.maxRequests = 15 * 60 * 2;
    addDefaults(co);
    return co;
  }
  
  private static CrawlerOptions addDefaults(CrawlerOptions co) {
    co.allowedDomains.add(".crawl.baidu.com");
    co.allowedDomains.add(".crawl.baidu.jp");
    co.allowedDomains.add(".search.msn.com");
    co.allowedDomains.add(".google.com");
    co.allowedDomains.add(".googlebot.com");
    co.allowedDomains.add(".crawl.yahoo.net");
    co.allowedDomains.add(".yandex.com");
    co.allowedDomains.add(".yandex.net");
    co.allowedDomains.add(".yandex.ru");
    co.badRequest.add(".*index.(php|asp).*");
    co.badRequest.add(".*phpmyadmin.*");
    co.badRequest.add(".*phpstorm.*");
    co.badRequest.add(".*phpunit.*");
    co.badRequest.add(".*crypto/\\.env.*");
    co.badRequest.add(".*invokefunction.*");
    co.badRequest.add(".*aws/credentials.*");
    co.badRequest.add(".*git/credentials.*");
    co.badRequest.add(".*Dockerrun.aws.json.*");
    co.badRequest.add(".*/cgi-bin/.*");
    co.badRequest.add(".*\\.well-known/security.txt.*");
    co.badAgent.add(".*MtmKilledYou.*");
    return co;
  }
  
  /* IJSONizable */
  
  @Override
  public String toJson() {
    return toJson(this);
  }
 
  @Override
  public String toJson(Map<String, Object> properties) {
    return toJson(this, properties);
  }
 
}