Create Enricher

import json
# this enricher add tag "small_file_size"
def enricher(enricher , records , utils=None):
    parameters  = json.loads(enricher['parameters']) # get the parameters of the enricher
     # this is the max size if the file size less it will consider it small file
    max_size    = parameters['extra']['max_size']   if 'max_size' in parameters['extra'].keys() else 1000000
    max_size    = int(max_size)
    tag         = parameters['extra']['tag']        if 'tag' in parameters['extra'].keys() else 'small_file_size'
    # iterate over all records
    for r in range(0 , len(records)):
        # get the file size from the record on field "_source.file.size"
        size = utils.json.get(j=records[r] , p='_source.file.size')
        # if the record has a field '_source.file.size' and the file size less than max_size then add tag to the record
        if size[0] and int(size[1]) < max_size:
            if 'tags' not in records[r]['_source'].keys():
                # if there is no tags field before, add tag as a list
                records[r]['_source']['tags'] = [tag]
            else:
                # if the tag field already exists, then add the tag to it.
                records[r]['_source']['tags'].append(tag)
    return records

def enricher(enricher , records , utils=None):
    chunk_size          = 500
    parameters          = json.loads(enricher['parameters'])
    enricher_index      = f"enricher_{enricher['id']}"
    try:
        for data_start in range(0 , len(records) , chunk_size):
            # get list of records to enrich from the records list
            data = records[data_start:data_start+chunk_size]
            if len(data) == 0:
                break
            # collect all keys to get it from ES if exists
            keys = []
            for d in data:
                if d['_source']['key'] not in keys:
                    keys.append(d['_source']['key'])
            # get the keys from cache first
            es_keys             = utils.es.cache_get(index=enricher_index , keys=keys)
            results_to_store    = {} # contains results as {key1: meta1, key2: meta2, ...}
            records_to_delete   = [] # contains list of record ids to be deleted from the case (type:`data`)
            # enrich the data records
            for d in data:
                # delete the enriched record
                records_to_delete.append(d['_id'])
                try:
                    results = utils.es.cache_validate(cache_set=es_keys , key=d['_source']['key'])
                    # if the record not found in the cache, pull the info from external source
                    if results is None:
                        results = fetch_from_external(record=d, utils= utils)
                        # add to results_to_store so that it will store results in cache.
                        results_to_store[d['_source']['key']] = results
                    if results is not None:
                        # build the enrichment information document
                        enrichment = {
                                        'description'   : f"Indicator {d['_source']['key']} enriched by {enricher['name']}",
                                        'type'          : 'file',      # list of https://www.elastic.co/guide/en/ecs/current/ecs-threat.html#field-threat-enrichments-indicator-type
                                        'name'          : d['_source']['key'],
                                        'provider'      : enricher['name'],
                                        'matched': {
                                            'field'     : d['_source']['src_field'].lstrip("_source."),
                                            'occurred'  : str(datetime.now()),
                                            'atomic'    : d['_source']['key']
                                        },
                                        'last_seen'     : results['last_seen'],
                                        'Details'       : {
                                            f"{enricher['name']}" : results
                                        },
                                    }
                        # build the tags to be added to the tags field
                        new_tags = ['detected']
                        # update the enricher field
                        results   = utils.es.update(index=d['_source']['rec_index'] , data=[enrichment] , id=d['_source']['rec_id'] , field_list='threat.enrichment.indicator', get_source=False)
                        # update the tags field
                        results   = utils.es.update(index=d['_source']['rec_index'] , data=new_tags , id=d['_source']['rec_id'] , field_list='tags', get_source=False)
                except Exception as e:
                    utils.logger.warning(f"Failed to enrich the record [{d['_source']['rec_id']}] in index [{d['_source']['rec_index']}]: {str(e)}")
            # delete records after enriching
            if len(records_to_delete):
                results             = utils.es.update(index=enricher_index , data={"delete": True} , id=records_to_delete, get_source=False)
            # store the collected results to cache
            if len(results_to_store.keys()):
                success_records , failed_records = utils.es.cache_set(index=enricher_index, key_meta=results_to_store)
                utils.logger.debug(f"New results from {enricher['name']} stored, Success:{success_records}, Failed: {failed_records}")
    except Exception as e:
        utils.logger.warning(f"Failed to enrich by [{enricher['name']}] enricher : {str(e)}")
    return True