Finding Anomalies In Your Keywords, Adgroups, and Ads

UPDATE 2013-07-23: Added the ability to add labels to your ads as well. This might be helpful for creative testing.

In a recent post from SearchEngineLand, Larry Kim wants PPC Managers to stop being lazy. Link baiting aside, he has some good points. I think much of the time spent pouring over client spreadsheets is really trying to answer the simple question of "Where Are The Anomalies?"

Anomalies are keywords or AdGroups that seem to be performing very differently than their brothers and sisters in the same AdGroup or Campaign. Maybe you have an AdGroup with 15 keywords and one or two keywords seem to be getting nearly all of the clicks. Or maybe you just have an AdGroup that seems to be eating up a good chunk of your budget and you want to have more control over it. In either of these cases, you need some way to quickly identify which Keywords or AdGroups you want to take action on.

So I created the following script to help me do just that. It has a little bit of statistics embedded in it to calculate the mean and standard deviation of a set of entities (AdGroups or Keywords). Then I apply a label to any entity that seems to be more than two standard deviations away from the mean, which would indicate it is performing much better or much worse than its siblings. That way, I can take action on those entities pretty easily in my account.

This script will also send you an email each day with a summary of the entities that it deems to be anomalies. Your goal is to fix these issues by moving them into their own Campaigns and AdGroups or maybe even getting rid of them altogether (possibly negatives).

Right now, the script will check every metric that is available in the AdWordsApp.stats object. It could be easily modified to check for things like cost per conversion or profit per impression. You will see in the code below where you can remove stats that don't interest you as well.

One note of warning is that I do not claim to be a statistics guy, so this may or may not be a valid way to look at things. Hopefully, it will help you locate the issues in an account quickly and fix them so that you can spend more time being lazy :)

Also, I am open to your comments and suggestions for this script. Is it useful to anyone else?

Thanks,
Russ

/**************************************
* Find the Anomalies
* Created By: Russ Savage
* Version: 1.2
* Changelog v1.2
*  - Fixed divide by 0 errors
*  - Changed SIG_FIGS to DECIMAL_PLACES
* Changelog v1.1
*  - Added ability to tag ad anomalies as well
* FreeAdWordsScripts.com
**************************************/
var DATE_RANGE = 'LAST_30_DAYS';
var DECIMAL_PLACES = 3;
var STANDARD_DEVIATIONS = 2;
var TO = ['you@your_domain.com'];
 
function main() {
  // This will add labels to and send emails about adgroups, keywords and ads. Remove any if you like.
  var levels_to_tag = ['adgroup','keyword','ad'];
  for(var x in levels_to_tag) {
    var report = getContentRows(levels_to_tag[x]);
    var entity_map = buildEntityMap(levels_to_tag[x]);
    for(var parent_id in entity_map) {
      var child_list = entity_map[parent_id];
      var stats_list = Object.keys(child_list[0].stats);
      for(var i in stats_list) {
        var mean = getMean(child_list,stats_list[i]);
        var stand_dev = getStandardDev(child_list,mean,stats_list[i]);
        var label_name = stats_list[i]+"_anomaly";
        report += addLabelToAnomalies(child_list,mean,stand_dev,stats_list[i],label_name,levels_to_tag[x]);
      }
    }
    sendResultsViaEmail(report,levels_to_tag[x]);
  }
}
  
//Takes a report and the level of reporting and sends and email
//with the report as an attachment.
function sendResultsViaEmail(report,level) {
  var rows = report.match(/\n/g).length - 1;
  if(rows == 0) { return; }
  var options = { attachments: [Utilities.newBlob(report, 'text/csv', level+"_anomalies_"+_getDateString()+'.csv')] };
  var email_body = "There are " + rows + " " + level + "s that have abnormal performance. See attachment for details.";
  var subject = 'Abnormal ' + _initCap(level) + ' Entities Report - ' + _getDateString();
  for(var i in TO) {
    MailApp.sendEmail(TO[i], subject, email_body, options);
  }
}
  
//Helper function to return a single row of the report formatted correctly
function toReportRow(entity,level,label_name) {
  var ret_val = [AdWordsApp.currentAccount().getCustomerId(),
                 entity.getCampaign().getName()];
  ret_val.push( (level == 'adgroup') ? entity.getName() : entity.getAdGroup().getName() );
  if(level == 'keyword') {
    ret_val = ret_val.concat([entity.getText(),entity.getMatchType()]); 
  } else if(level == 'ad') {
    ret_val = ret_val.concat([entity.getHeadline(),entity.getDescription1(),entity.getDescription2(),entity.getDisplayUrl()]); 
  }
  ret_val.push(label_name);
  return '"' + ret_val.join('","') + '"\n';
}
  
//Helper function to return the column headings for the report
function getContentRows(level) {
  var ret_val = ['AccountId','CampaignName','AdGroupName'];
  if(level == 'keyword') {
    ret_val = ret_val.concat(['KeywordText','MatchType']); 
  } else if(level == 'ad') {
    ret_val = ret_val.concat(['Headline','Description1','Description2','DisplayUrl']);
  }
  ret_val.push('LabelName');
  return '"' + ret_val.join('","') + '"\n';
}
  
//Function to add the labels to the entities based on the standard deviation and mean.
//It returns a csv formatted string for reporting
function addLabelToAnomalies(entites,mean,sd,stat_key,label_name,level) {
  createLabelIfNeeded(label_name);
  var report = '';
  for(var i in entites) {
    var entity = entites[i]['entity'];
    var deviation = Math.abs(entites[i]['stats'][stat_key] - mean);
    if(sd != 0 && deviation/sd >= STANDARD_DEVIATIONS) {
      entity.applyLabel(label_name);
      report += toReportRow(entity,level,label_name);
    } else {
      entity.removeLabel(label_name);
    }
  }
  return report;
}
  
//This is a helper function to create the label if it does not already exist
function createLabelIfNeeded(name) {
  if(!AdWordsApp.labels().withCondition("Name = '"+name+"'").get().hasNext()) {
    AdWordsApp.createLabel(name);
  }
}
  
//This function returns the standard deviation for a set of entities
//The stat key determines which stat to calculate it for
function getStandardDev(entites,mean,stat_key) {
  var total = 0;
  for(var i in entites) {
    total += Math.pow(entites[i]['stats'][stat_key] - mean,2);
  }
  if(Math.sqrt(entites.length-1) == 0) {
    return 0;
  }
  return round(Math.sqrt(total)/Math.sqrt(entites.length-1));
}
  
//Returns the mean (average) for the set of entities
//Again, stat key determines which stat to calculate this for
function getMean(entites,stat_key) {
  var total = 0;
  for(var i in entites) {
    total += entites[i]['stats'][stat_key];
  }
  if(entites.length == 0) {
    return 0;
  }
  return round(total/entites.length);
}
  
//This function returns a map of the entities that I am processing.
//The format for the map can be found on the first line.
//It is meant to work on AdGroups and Keywords
function buildEntityMap(entity_type) {
  var map = {}; // { parent_id : [ { entity : entity, stats : entity_stats } ], ... }
  var iter = getIterator(entity_type);
  while(iter.hasNext()) {
    var entity = iter.next();
    var stats = entity.getStatsFor(DATE_RANGE);
    var stats_map = getStatsMap(stats);
    var parent_id = getParentId(entity_type,entity);
    if(map[parent_id]) { 
      map[parent_id].push({entity : entity, stats : stats_map});
    } else {
      map[parent_id] = [{entity : entity, stats : stats_map}];
    }
  }
  return map;
}
  
//Given an entity type (adgroup or keyword) this will return the parent id
function getParentId(entity_type,entity) {
  switch(entity_type) {
    case 'adgroup' :
      return entity.getCampaign().getId();
    case 'keyword':
      return entity.getAdGroup().getId();
    case 'ad':
      return entity.getAdGroup().getId();
  }
}
  
//Given an entity type this will return the iterator for that.
function getIterator(entity_type) {
  switch(entity_type) {
    case 'adgroup' :
      return AdWordsApp.adGroups().forDateRange(DATE_RANGE).withCondition("Impressions > 0").get();
    case 'keyword' :
      return AdWordsApp.keywords().forDateRange(DATE_RANGE).withCondition("Impressions > 0").get();
    case 'ad' :
      return AdWordsApp.ads().forDateRange(DATE_RANGE).withCondition("Impressions > 0").get();
  }
}
  
//This returns a map of all the stats for a given entity.
//You can comment out the things you don't really care about.
function getStatsMap(stats) {
  return { // You can comment these out as needed.
          avg_cpc : stats.getAverageCpc(),
          avg_cpm : stats.getAverageCpm(),
          avg_pv : stats.getAveragePageviews(),
          avg_pos : stats.getAveragePosition(),
          avg_tos : stats.getAverageTimeOnSite(),
          bounce : stats.getBounceRate(),
          clicks : stats.getClicks(),
          cv : stats.getConversionRate(),
          conv : stats.getConversions(),
          cost : stats.getCost(),
          ctr : stats.getCtr(),
          imps : stats.getImpressions()
         };
}
  
//Helper function to format todays date
function _getDateString() {
  return Utilities.formatDate((new Date()), AdWordsApp.currentAccount().getTimeZone(), "yyyy-MM-dd");
}
  
//Helper function to capitalize the first letter of a string.
function _initCap(str) {
  return str.replace(/(?:^|\s)\S/g, function(a) { return a.toUpperCase(); });
}

// A helper function to make rounding a little easier
function round(value) {
  var decimals = Math.pow(10,DECIMAL_PLACES);
  return Math.round(value*decimals)/decimals;
}

Store Keyword Performance Report In Amazon S3

It's been a few weeks since my last post about putting data into Amazon S3, and in it, I mentioned that you could use it to store keyword performance reports for multiple AdWords Accounts in a single place. Here is an updated version which does exactly that. It combines my previous post about storing AdWords Account Performance Report in a Google Spreadsheet (with some changes of course).

Also, with this post, I will start versioning all of my scripts so that you can be sure you always have the latest version.

Thanks,
Russ

//-----------------------------------
// Store Keyword Performance Report in Amazon S3
// Created By: Russ Savage
// Version: 1.0
// FreeAdWordsScripts.com
//-----------------------------------
var ACCESS_KEY = 'YOUR_ACCESS_KEY_HERE';
var SECRET_KEY = 'YOUR_SECRET_KEY_HERE';
var S3_BUCKET = 'YOUR_S3_BUCKET_NAME_HERE';

function main() {
  var date_str = Utilities.formatDate(new Date(),AdWordsApp.currentAccount().getTimeZone(),'yyyy-MM-dd');
  var file_name = 'adwords_keyword_perf_'+AdWordsApp.currentAccount().getCustomerId() + '_' + date_str+'.csv';
  putDataToBucket(S3_BUCKET,'/'+file_name,getKeywordPerformanceReport());
}

function putDataToBucket(bucket,file_path,data) {
  var auth_options = {  
    method : 'PUT',
    base_url : "http://" + bucket + ".s3.amazonaws.com",
    s3_bucket : bucket,
    path : file_path,
    headers : { 
       "Date" : getDate(),
       "Content-Type" : "application/x-www-form-urlencoded"
    },
  };
  
  var auth_string = generateAuthString(auth_options);
  auth_options.headers["Authorization"] = auth_string;
  
  var options = {
    method : auth_options.method,
    headers : auth_options.headers,
    payload : data
  };
  
  return (UrlFetchApp.fetch(auth_options.base_url+auth_options.path, options).getResponseCode() == 200);
}

function generateAuthString(url) {
  var string_to_sign =  getStringToSign(url);
  Logger.log(string_to_sign);
  var signature = getSignature(SECRET_KEY,string_to_sign);
  return "AWS" + " " + ACCESS_KEY + ":" + signature;
}

function getSignature(SECRET_KEY,string_to_sign) {
  return  Utilities.base64Encode(
    Utilities.computeHmacSignature(
      Utilities.MacAlgorithm.HMAC_SHA_1,
      string_to_sign,
      SECRET_KEY,
      Utilities.Charset.UTF_8
    )
  );
}

function getStringToSign(url,params) {
  var method = url.method;
  var date = url.headers.Date;
  
  return method + "\n" + "\n" + 
    url.headers['Content-Type'] + "\n" +
    date + "\n" + 
    getCanonicalizedAmzHeaders(url);
}

function getCanonicalizedAmzHeaders(url) {
  var ret_val = "/" + url.s3_bucket;
  ret_val += url.path;
  return ret_val;
}

function getDate() {
  return Utilities.formatDate(new Date(),"GMT", "EEE, dd MMM yyyy HH:mm:ss +0000");
}


function getKeywordPerformanceReport() {
  var date_range = 'LAST_7_DAYS';
   
  var columns = ['Date',
                 'CampaignName',
                 'CampaignStatus',
                 'AdGroupName',
                 'AdGroupStatus',
                 'IsNegative',
                 'Id',
                 'KeywordMatchType',
                 'KeywordText',
                 'DestinationUrl',
                 'FirstPageCpc',
                 'MaxCpc',
                 'MaxCpm',
                 'PercentCpa',
                 'ClickType',
                 'Device',
                 'Slot',
                 'CpcBidSource',
                 'AverageCpc',
                 'AverageCpm',
                 'AveragePosition',
                 'PreferredPosition',
                 'QualityScore',
                 'Clicks',
                 'ConversionRate',
                 'ConversionRateManyPerClick',
                 'Conversions',
                 'ConversionsManyPerClick',
                 'ConversionValue',
                 'Cost',
                 'CostPerConversion',
                 'CostPerConversionManyPerClick',
                 'Ctr',
                 'Impressions',
                 'Cost',
                 'ValuePerConversion',
                 'ValuePerConversionManyPerClick',
                 'ViewThroughConversions'];
  var columns_str = columns.join(',') + " ";
   
  var report_iter = AdWordsApp.report(
    'SELECT ' + columns_str +
    'FROM KEYWORDS_PERFORMANCE_REPORT ' +
    'DURING ' + date_range, {
      includeZeroImpressions: false,
      apiVersion: 'v201302'
    }).rows();
  
  var ret_data = '"' + columns.join('","') + '"\n';
  while(report_iter.hasNext()) {
    var row = report_iter.next();
    var row_array = [];
    for(var i in columns) {
       row_array.push(row[columns[i]]);
    }
    ret_data += '"' + row_array.join('","') + '"\n';
  }
  
  return ret_data;
}

Put AdWords Data to Amazon S3 Using Scripts

I'm not sure about you, but writing all these scripts is putting me on Google Spreadsheet overload. With the release of the ability to access any of the AdWords reports, the data quickly starts to get unwieldy. Once things get over a few thousand rows, I much prefer to use Excel or store and manipulate data in a database.

So I was looking for other ways to get data from an AdWords report out of the scripts world and I thought of Amazon S3. Amazon offers a RESTful API through their Amazon Web Services that allows you to store and retrieve large amounts of data from the cloud. I figured I could use them to store some CSV files of account/campaign/keyword performance for downloading or processing later, maybe by another script or software.

The script below tries to encapsulate all the logic required to build and authorize a proper S3 put request. I followed the documentation found here.

To get started for free, sign up for Amazon S3 here and find your access and secret keys here:


Next, create a new S3 bucket to hold all your data through the AWS console. Fill in the details below and try a test file. Ideally, you would combine this with one of the other AdWords reporting scripts found here to start storing your data in the cloud.

NOTE: This is a very specific set of functions which represent the bare minimum required information to PUT a file to Amazon S3. For more information, please check out the full Amazon S3 Docs.

Thanks,
Russ

//-----------------------------------
// Put Data To Amazon S3
// Created By: Russ Savage
// FreeAdWordsScripts.com
//-----------------------------------
var ACCESS_KEY = 'YOUR_ACCESS_KEY_HERE';
var SECRET_KEY = 'YOUR_SECRET_KEY_HERE';
var S3_BUCKET = 'YOUR_S3_BUCKET_NAME_HERE';
  

function main() {
  var date_str = Utilities.formatDate(new Date(),AdWordsApp.currentAccount().getTimeZone(),'yyyy-MM-dd');
  var file_name = 'adwords_keyword_perf_'+date_str+'.csv';
  putDataToBucket(S3_BUCKET,'/'+file_name,'this is where the data from an AdWords report would go.');
}


function putDataToBucket(bucket,file_path,data) {
  var auth_options = {  
    method : 'PUT',
    base_url : "http://" + bucket + ".s3.amazonaws.com",
    s3_bucket : bucket,
    path : file_path,
    headers : { 
       "Date" : getDate(),
       "Content-Type" : "application/x-www-form-urlencoded"
    },
  };
  
  var auth_string = generateAuthString(auth_options);
  auth_options.headers["Authorization"] = auth_string;
  
  var options = {
    method : auth_options.method,
    headers : auth_options.headers,
    payload : data
  };
  
  return (UrlFetchApp.fetch(auth_options.base_url+auth_options.path, options).getResponseCode() == 200);
}

//Generates an AWS Auth String
//For more info, see the AWS docs - http://goo.gl/m5nCe
function generateAuthString(url) {
  var string_to_sign =  getStringToSign(url);
  Logger.log(string_to_sign);
  var signature = getSignature(SECRET_KEY,string_to_sign);
  return "AWS" + " " + ACCESS_KEY + ":" + signature;
}

//Generates an AWS Signature
//For more info, see the AWS docs - http://goo.gl/m5nCe
function getSignature(SECRET_KEY,string_to_sign) {
  return  Utilities.base64Encode(
    Utilities.computeHmacSignature(
      Utilities.MacAlgorithm.HMAC_SHA_1,
      string_to_sign,
      SECRET_KEY,
      Utilities.Charset.UTF_8
    )
  );
}

//Generates an AWS string to sign
//For more info, see the AWS docs - http://goo.gl/m5nCe
function getStringToSign(url,params) {
  var method = url.method;
  var date = url.headers.Date;
  
  return method + "\n" + "\n" + 
    url.headers['Content-Type'] + "\n" +
    date + "\n" + 
    getCanonicalizedAmzHeaders(url);
}

//Generates the Canonicalized Amazon Headers (not really)
//For more info, see the AWS docs - http://goo.gl/m5nCe
function getCanonicalizedAmzHeaders(url) {
  var ret_val = "/" + url.s3_bucket;
  ret_val += url.path;
  return ret_val;
}

function getDate() {
  return Utilities.formatDate(new Date(),"GMT", "EEE, dd MMM yyyy HH:mm:ss +0000");
}

Put Current Crime Statistics in Your Creatives

I live in Chicago and one of the things this city is known for is it's colorful history of Organized Crime. So I thought it might be interesting to see if I could use that history to my advantage.

Chicago runs a website that collects and aggregates the statistics of the city: City of Chicago Data Portal. Here you can find all sorts of data sets, but the one I was interested in was the Crime Stats.

After a little exploration, I found that the data is available in JSON (an easy to consume format for software). So I put together a script to read that data and then automatically update the keyword AdParams with the total count of the different types of crime.

This might come in handy for any companies that sell insurance or security systems, but I'm sure there's other companies that might be able to use this info.

Other major cities in the US keep similar sets of data as well:

• New York, NY Crime Data
• Seattle, WA Crime Data
• A listing of all datasets

What other public records data could your business use to target your customers? Let me know in the comments and I'll take a look.

Thanks,
Russ

//-----------------------------------
// Put Chicago Crime Stats in Your Creatives
// Created By: Russ Savage
// FreeAdWordsScripts.com
//-----------------------------------
function main() {
  // You can get this link by going here: http://goo.gl/tfNgM
  // Apply some filters and then click export > api
  // This end point is good for all of 2013
  var DATA_ENDPOINT = "http://data.cityofchicago.org/resource/pga9-zdiw.json";
  var CAMPAIGN_PREFIX = 'Crime_Data_Chicago_'; //All your campaigns start with this
  var AD_PARAM = 1; // 1 or 2
  
  try {
    var json = Utilities.jsonParse(UrlFetchApp.fetch(DATA_ENDPOINT).getContentText());
    var summary = summarizeCrimeStats(json);
    //logCrimeSummary(summary);
    
    for(var i in summary) {
      var total = totalPrimaryDescription(i,summary);
      var kw_iter = AdWordsApp.keywords()
                      .withCondition("CampaignName CONTAINS_IGNORE_CASE '"+CAMPAIGN_PREFIX+i+"'")
                      .get();
      while(kw_iter.hasNext()) {
        var kw = kw_iter.next();
        kw.setAdParam(AD_PARAM, total);
      }
    }
  }catch(e) {
  }
}

// A helper function to aggregate the data by primary description
function totalPrimaryDescription(key,summary) {
  var tot = 0;
  for(var i in summary[key]) {
    tot += summary[key][i];
  }
  return tot;
}

//This function takes in a json formatted object and stores the count of instances
//in a 2 dimentional hash of [Primary Description][Secondary Description]
function summarizeCrimeStats(json) {
  var crime_summary = {};
  for(var i in json) {
    var crime = json[i];
    if(crime_summary[crime._primary_decsription]) {
      if(crime_summary[crime._primary_decsription][crime._secondary_description]) {
        crime_summary[crime._primary_decsription][crime._secondary_description]++;
      }else{
        crime_summary[crime._primary_decsription][crime._secondary_description] = 1;
      }
    }else{
      crime_summary[crime._primary_decsription] = {};
      crime_summary[crime._primary_decsription][crime._secondary_description] = 1;
    }
  }
  return crime_summary;
}

//Just a helper function to print out the summary info so that
//I can find the data I'm interested in.
function logCrimeSummary(crime_summary) {
  for(var i in crime_summary) {
    for(var x in crime_summary[i]) {
      Logger.log([i,x,crime_summary[i][x]].join(', '));
    }
  }
}