Production script in Google for Anthology. Currently seems to be working well.
This commit is contained in:
Norm Rasmussen
1
Scripts/GAS_GS/AnthologyCSV-Parse-Prod/.clasp.json
Normal file
1
Scripts/GAS_GS/AnthologyCSV-Parse-Prod/.clasp.json
Normal file
@ -0,0 +1 @@
|
||||
{"scriptId":"1daBpFcVj0FuYrB6TJjXJX3YAVHZcK-BcHXJSW6kdngRcEKitCxbhncbe","rootDir":"/Users/normrasmussen/Documents/Work/Scripts/GAS_GS/AnthologyCSV-Parse-Prod"}
|
||||
251
Scripts/GAS_GS/AnthologyCSV-Parse-Prod/MasterCSV_Parser.js
Normal file
251
Scripts/GAS_GS/AnthologyCSV-Parse-Prod/MasterCSV_Parser.js
Normal file
@ -0,0 +1,251 @@
|
||||
const apiKey = '8ALsk8jDOlynEwn8ScMBSnG87';
|
||||
|
||||
function newTrigger() {
|
||||
ScriptApp.newTrigger(readLog)
|
||||
.timeBased()
|
||||
.everyMinutes(1)
|
||||
.create();
|
||||
prepareSheet();
|
||||
}
|
||||
|
||||
async function prepareSheet() {
|
||||
var ss = SpreadsheetApp.openByUrl('https://docs.google.com/spreadsheets/d/1Kck1UUOkVIU4kbBC8b_bl77fNaCTSFStiv0I5Gg-kIc/');
|
||||
sheet = ss.getActiveSheet();
|
||||
sheet = ss.getActiveSheet();
|
||||
lastRow = sheet.getLastRow();
|
||||
sheet.clear()
|
||||
sheet.getRange(1,1).setValue("domain");
|
||||
for (var sh = 1; sh < 100; sh++) {
|
||||
sheet.getRange(1, sh+1).setValue("groups_"+sh)
|
||||
}
|
||||
}
|
||||
|
||||
function readLog() {
|
||||
//var previousLast = 0
|
||||
var folder = DriveApp.getFolderById("1nAkWPYFNpRx-UZECMJN4C3N--i0Clcho")
|
||||
var files = folder.getFiles()
|
||||
while (files.hasNext()) {
|
||||
var fileName = files.next()
|
||||
if (fileName == "status.json") {
|
||||
var content = fileName.getBlob().getDataAsString();
|
||||
var json = JSON.parse(content);
|
||||
var previousLast = json['lastLine'];
|
||||
main(previousLast)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createLog() {
|
||||
var folder = DriveApp.getFolderById("1hAz7O-eoxFUk4YW7FoHr4uNJmPs7CiV5BjBNTLtMJzXwY0CbpykYgSAkFR5Quy_MPdn3e_5j")
|
||||
blob = JSON.stringify(obj);
|
||||
file = folder.createFile("status.json", blob);
|
||||
}
|
||||
|
||||
function updateLog(lastUpdatedLine) {
|
||||
var folder = DriveApp.getFolderById("1hAz7O-eoxFUk4YW7FoHr4uNJmPs7CiV5BjBNTLtMJzXwY0CbpykYgSAkFR5Quy_MPdn3e_5j");
|
||||
obj = JSON.stringify({ lastLine: lastUpdatedLine });
|
||||
var files = folder.getFiles();
|
||||
while (files.hasNext()) {
|
||||
var fileName = files.next();
|
||||
if (fileName == "status.json") {
|
||||
Logger.log(fileName+" found!")
|
||||
fileName.setContent(obj);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function deleteLog() {
|
||||
var folder = DriveApp.getFolderById("1hAz7O-eoxFUk4YW7FoHr4uNJmPs7CiV5BjBNTLtMJzXwY0CbpykYgSAkFR5Quy_MPdn3e_5j");
|
||||
var files = folder.getFiles();
|
||||
while (files.hasNext()) {
|
||||
var fileName = files.next();
|
||||
if (fileName == "status.json") {
|
||||
folder.removeFile(fileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function main(previousLast) {
|
||||
Logger.log("Main PL: "+previousLast)
|
||||
var folder = DriveApp.getFolderById("1hAz7O-eoxFUk4YW7FoHr4uNJmPs7CiV5BjBNTLtMJzXwY0CbpykYgSAkFR5Quy_MPdn3e_5j");
|
||||
var files = folder.getFiles();
|
||||
while (files.hasNext()) {
|
||||
var file = files.next();
|
||||
var getDate = new Date();
|
||||
var date = Utilities.formatDate(getDate, 'America/New_York', 'MM/dd/yyyy hh');
|
||||
var fileDate = file.getDateCreated();
|
||||
var fileDateFormatted = Utilities.formatDate(fileDate, 'America/New_York', 'MM/dd/yyyy hh');
|
||||
if (date == fileDateFormatted) {
|
||||
var updated_file = file.getBlob()
|
||||
var updated_blob = updated_file.getDataAsString();
|
||||
var updated_data = Utilities.parseCsv(updated_blob, ',');
|
||||
//var updated_data = updated_data.slice(previousLast);
|
||||
var full_data = Utilities.parseCsv(updated_data, ',');
|
||||
//var full_data = full_data.slice(previousLast)
|
||||
} else if (date != fileDateFormatted) {
|
||||
var outdated_file = file.getBlob()
|
||||
var outdated_blob = outdated_file.getDataAsString();
|
||||
var outdated_data = Utilities.parseCsv(outdated_blob, ',');
|
||||
//var outdated_data = outdated_data.slice(previousLast);
|
||||
}
|
||||
}
|
||||
austinComparseData(outdated_data, updated_data);
|
||||
fullParse(full_data);
|
||||
moveOldFiles();
|
||||
}
|
||||
|
||||
function moveOldFiles() {
|
||||
var folder = DriveApp.getFolderById("15p4vXVQd6hhcMl73VOxD8VctL2gR-q43")
|
||||
var files = folder.getFiles()
|
||||
while (files.hasNext()) {
|
||||
var file = files.next();
|
||||
var getDate = new Date();
|
||||
var date = Utilities.formatDate(getDate, 'America/New_York', 'MM/dd/yyyy');
|
||||
var fileDate = file.getDateCreated();
|
||||
var fileDateFormatted = Utilities.formatDate(fileDate, 'America/New_York', 'MM/dd/yyyy');
|
||||
if (date != fileDateFormatted) {
|
||||
const destination = DriveApp.getFolderById('17WCI5QkV9W6j35a-1n9BspTF6z_MZ9yu');
|
||||
file.moveTo(destination);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function austinComparseData(outdated_data, updated_data) {
|
||||
let domains_to_update = [];
|
||||
let updatedFlag = 1;
|
||||
let outdatedFlag = 1;
|
||||
|
||||
while(updatedFlag < updated_data.length && outdatedFlag < outdated_data.length){
|
||||
if(updated_data[updatedFlag][0] == outdated_data[outdatedFlag][0]) {
|
||||
let allUpdatedGroups = [];
|
||||
let allOutdatedGroups = [];
|
||||
|
||||
for(let i = 1; i < updated_data[updatedFlag].length; i++){
|
||||
if(updated_data[updatedFlag][i]){
|
||||
allUpdatedGroups.push(updated_data[updatedFlag][i].trim());
|
||||
}
|
||||
}
|
||||
for(let j = 1; j < outdated_data[outdatedFlag].length; j++){
|
||||
if(outdated_data[outdatedFlag][j]){
|
||||
allOutdatedGroups.push(outdated_data[outdatedFlag][j].trim());
|
||||
}
|
||||
}
|
||||
|
||||
if(allUpdatedGroups.length !== allOutdatedGroups.length){
|
||||
domains_to_update.push(updated_data[updatedFlag]);
|
||||
} else {
|
||||
for(let k = 0; k < allUpdatedGroups.length; k++){
|
||||
if(allOutdatedGroups.indexOf(allUpdatedGroups[k]) == -1){
|
||||
domains_to_update.push(updated_data[updatedFlag]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
updatedFlag++;
|
||||
outdatedFlag++;
|
||||
} else {
|
||||
if(updated_data[updatedFlag][0] > outdated_data[outdatedFlag][0]){
|
||||
outdatedFlag++;
|
||||
}
|
||||
else if(updated_data[updatedFlag][0] < outdated_data[outdatedFlag][0]){
|
||||
domains_to_update.push(updated_data[updatedFlag]);
|
||||
updatedFlag++;
|
||||
}
|
||||
}
|
||||
|
||||
if(updatedFlag > updated_data.length-1 || outdatedFlag > outdated_data.length-1){
|
||||
break;
|
||||
}
|
||||
}
|
||||
updatedDomainsOnly(domains_to_update);
|
||||
}
|
||||
|
||||
function updatedDomainsOnly(domains_to_update) {
|
||||
for (var x = 0; x < domains_to_update.length; x++) {
|
||||
var array = domains_to_update[x];
|
||||
for (var i = 1; i < array.length; i++) {
|
||||
var item = array[i]
|
||||
if (item != "") {
|
||||
var api_url = 'https://api.northpass.com/v2/groups/?filter[name][eq]='+encodeURIComponent(item);
|
||||
const settings = {
|
||||
async: true,
|
||||
crossDomain: true,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
'X-Api-Key': apiKey
|
||||
}
|
||||
};
|
||||
const sendMsg = UrlFetchApp.fetch(api_url, settings);
|
||||
var uuidResponse = sendMsg.getContentText();
|
||||
var parseData = JSON.parse(uuidResponse);
|
||||
var groupID = parseData['data'][0]['id'];
|
||||
var groupName = array.indexOf(item);
|
||||
if (groupName != -1) {
|
||||
array[groupName] = groupID;
|
||||
}
|
||||
}
|
||||
}
|
||||
const dom = array.shift();
|
||||
const groups = array;
|
||||
sendWebhook({domain_to_update : { domain: dom, group_ids: groups }})
|
||||
}
|
||||
}
|
||||
|
||||
function fullParse(full_data) {
|
||||
for (var x = 1; x < full_data.length; x++) {
|
||||
var array = full_data[x];
|
||||
for (var i = 1; i < array.length; i++) {
|
||||
var item = array[i]
|
||||
if (item != "") {
|
||||
var api_url = 'https://api.northpass.com/v2/groups/?filter[name][eq]='+encodeURIComponent(item);
|
||||
const settings = {
|
||||
async: true,
|
||||
crossDomain: true,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
'X-Api-Key': apiKey
|
||||
}
|
||||
};
|
||||
const sendMsg = UrlFetchApp.fetch(api_url, settings);
|
||||
var uuidResponse = sendMsg.getContentText();
|
||||
var parseData = JSON.parse(uuidResponse)
|
||||
Logger.log("Current Check:"+item)
|
||||
var groupID = parseData['data'][0]['id']
|
||||
var groupName = array.indexOf(item);
|
||||
if (groupName != -1) {
|
||||
array[groupName] = groupID;
|
||||
}
|
||||
}
|
||||
}
|
||||
const dom = array.shift();
|
||||
const groups = array;
|
||||
previousLast++
|
||||
Logger.log(previousLast)
|
||||
updateLog(previousLast)
|
||||
writeDataToSheet(dom, groups)
|
||||
}
|
||||
}
|
||||
|
||||
//Inserts a new sheet and writes a 2D array of data in it
|
||||
function writeDataToSheet(dom, groups) {
|
||||
var ss = SpreadsheetApp.openByUrl('https://docs.google.com/spreadsheets/d/1Kck1UUOkVIU4kbBC8b_bl77fNaCTSFStiv0I5Gg-kIc/');
|
||||
var dataRow = Math.max(sheet.getLastRow(),1);
|
||||
sheet.insertRowAfter(dataRow);
|
||||
sheet.getRange(dataRow + 1, 1).setValue(dom);
|
||||
for (var g = 1; g < groups.length; g++) {
|
||||
sheet.getRange(dataRow + 1, g+1).setValue(groups[g]);
|
||||
}
|
||||
}
|
||||
|
||||
function sendWebhook(domain_to_update){
|
||||
Logger.log("Webhook function Object: "+JSON.stringify(domain_to_update))
|
||||
var payload = domain_to_update
|
||||
var options = {
|
||||
'method': 'post',
|
||||
'payload': JSON.stringify(payload)
|
||||
};
|
||||
UrlFetchApp.fetch('https://webhooks.workato.com/webhooks/rest/604b4fc0-fcd9-4f3b-b715-b42d7f740ba6/domains-updated', options);
|
||||
|
||||
}
|
||||
7
Scripts/GAS_GS/AnthologyCSV-Parse-Prod/appsscript.json
Normal file
7
Scripts/GAS_GS/AnthologyCSV-Parse-Prod/appsscript.json
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"timeZone": "America/New_York",
|
||||
"dependencies": {
|
||||
},
|
||||
"exceptionLogging": "STACKDRIVER",
|
||||
"runtimeVersion": "V8"
|
||||
}
|
||||
48
Scripts/list_deliveries_try_it.json
Normal file
48
Scripts/list_deliveries_try_it.json
Normal file
@ -0,0 +1,48 @@
|
||||
{
|
||||
"links": {
|
||||
"self": "https://api.northpass.com/v1/communications/deliveries?include=person"
|
||||
},
|
||||
"data": [
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "c339a2aa-3a55-4181-ab66-1d9d19eabfb8",
|
||||
"attributes": {
|
||||
"communication_id": "account_activation_learners",
|
||||
"created_at": "2024-01-09T06:22:15Z"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "362a0113-86ea-4568-9778-78bf52e7956e"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"included": [
|
||||
{
|
||||
"type": "people",
|
||||
"id": "362a0113-86ea-4568-9778-78bf52e7956e",
|
||||
"attributes": {
|
||||
"activated_at": "2024-01-09T06:22:15Z",
|
||||
"created_at": "2024-01-09T06:22:15Z",
|
||||
"email": "test+user-60-1704781335@schoolkeep.com",
|
||||
"updated_at": "2024-01-09T06:22:15Z",
|
||||
"unsubscribed": false
|
||||
},
|
||||
"links": {
|
||||
"self": "https://api.northpass.com/v1/people/362a0113-86ea-4568-9778-78bf52e7956e",
|
||||
"teaching": "https://app.northpass.com/people/362a0113-86ea-4568-9778-78bf52e7956e"
|
||||
},
|
||||
"relationships": {
|
||||
"school": {
|
||||
"data": {
|
||||
"type": "schools",
|
||||
"id": "9f22f5b8-2721-4b2b-a1a0-882642149495"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
189
Scripts/list_deliviers_api.json
Normal file
189
Scripts/list_deliviers_api.json
Normal file
@ -0,0 +1,189 @@
|
||||
{
|
||||
"links": {
|
||||
"self": "https://api.northpass.com/v1/communications/deliveries",
|
||||
"next": "https://api.northpass.com/v1/communications/deliveries?page=2"
|
||||
},
|
||||
"data": [
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "006b8a25-6d7f-426b-b231-efdd93db8802",
|
||||
"attributes": {
|
||||
"communication_id": "certificate_awarded",
|
||||
"created_at": "2024-02-02T17:03:29Z",
|
||||
"delivered_at": "2024-02-02T17:03:33Z",
|
||||
"status": "delivered"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "d4194ec6-d9fc-4a15-b163-4f3b6d927c79"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "2e4bb0f8-9f3e-4e0a-95b2-53a896a81c2c",
|
||||
"attributes": {
|
||||
"communication_id": "certificate_awarded",
|
||||
"created_at": "2024-02-02T16:38:25Z",
|
||||
"delivered_at": "2024-02-02T16:38:27Z",
|
||||
"status": "delivered"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "fdae3c20-ee70-4aac-8ea8-f4472ac98d56"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "4ee2ad2c-ff43-4c78-8cbd-7232a19b5c14",
|
||||
"attributes": {
|
||||
"communication_id": "courses_incomplete_notification",
|
||||
"created_at": "2024-02-02T15:23:51Z",
|
||||
"status": "failed"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "0e4b602d-952d-4ad8-9d64-dadbc87b3cf9"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "5c821046-9e46-415c-8771-fcb72c5a7bbb",
|
||||
"attributes": {
|
||||
"communication_id": "courses_incomplete_notification",
|
||||
"created_at": "2024-02-02T15:23:50Z",
|
||||
"status": "failed"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "0e4b602d-952d-4ad8-9d64-dadbc87b3cf9"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "929f031b-c5d2-4d00-b42d-85fda56b24eb",
|
||||
"attributes": {
|
||||
"communication_id": "certificate_awarded",
|
||||
"created_at": "2024-02-02T13:52:53Z",
|
||||
"delivered_at": "2024-02-02T13:52:57Z",
|
||||
"status": "delivered"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "4f123cbd-aba5-4684-ba1c-c7fe23e8e306"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "7a22aa62-487e-4c50-b91a-3d2b9c9f88ea",
|
||||
"attributes": {
|
||||
"communication_id": "certificate_awarded",
|
||||
"created_at": "2024-02-02T12:44:08Z",
|
||||
"delivered_at": "2024-02-02T12:44:11Z",
|
||||
"status": "delivered"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "f99a3fa1-d373-4f95-ab39-9d4b4e716a6c"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "b4eec72e-ee50-4427-bb96-60ec8bb516f2",
|
||||
"attributes": {
|
||||
"communication_id": "courses_incomplete_notification",
|
||||
"created_at": "2024-02-02T03:29:21Z",
|
||||
"delivered_at": "2024-02-02T03:29:24Z",
|
||||
"opened_at": "2024-02-02T14:42:13Z",
|
||||
"status": "open"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "02af2df0-4727-4de9-af92-956cbc989738"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "2f9306d7-b4bc-44ed-bb49-3b088cb5c1bd",
|
||||
"attributes": {
|
||||
"communication_id": "courses_incomplete_notification",
|
||||
"created_at": "2024-02-02T03:27:56Z",
|
||||
"delivered_at": "2024-02-02T03:27:58Z",
|
||||
"opened_at": "2024-02-02T03:43:41Z",
|
||||
"status": "open"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "aef9e208-bfe2-4871-9723-4a69e966db19"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "3d9af6f9-da85-48cc-9dab-67c232b39da4",
|
||||
"attributes": {
|
||||
"communication_id": "courses_incomplete_notification",
|
||||
"created_at": "2024-02-02T03:27:55Z",
|
||||
"delivered_at": "2024-02-02T03:27:57Z",
|
||||
"status": "delivered"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "9ff19205-5243-4a1f-ad35-853364b073a0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "communication_activities",
|
||||
"id": "c44110af-7366-4e8a-8c81-7592748ec6ee",
|
||||
"attributes": {
|
||||
"communication_id": "courses_incomplete_notification",
|
||||
"created_at": "2024-02-02T03:27:54Z",
|
||||
"delivered_at": "2024-02-02T03:27:56Z",
|
||||
"opened_at": "2024-02-02T07:51:03Z",
|
||||
"status": "open"
|
||||
},
|
||||
"relationships": {
|
||||
"person": {
|
||||
"data": {
|
||||
"type": "people",
|
||||
"id": "0e86da6f-2ad4-4647-ac39-f962e144c59e"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user