Backed up Williams-Sonoma Google Script
This commit is contained in:
Norm Rasmussen
16
Scripts/GAS_GS/Williams-Sonoma-Property-Scraper/.clasp.json
Normal file
16
Scripts/GAS_GS/Williams-Sonoma-Property-Scraper/.clasp.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"scriptId": "1GO6XQKg9AiO-CDXBNGZaAbznyXj--TzMBlcZBUe3cKU6Xnw-Vo6yQFhY",
|
||||
"rootDir": "",
|
||||
"scriptExtensions": [
|
||||
".js",
|
||||
".gs"
|
||||
],
|
||||
"htmlExtensions": [
|
||||
".html"
|
||||
],
|
||||
"jsonExtensions": [
|
||||
".json"
|
||||
],
|
||||
"filePushOrder": [],
|
||||
"skipSubdirectories": false
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
{
|
||||
"timeZone": "America/New_York",
|
||||
"dependencies": {
|
||||
},
|
||||
"exceptionLogging": "STACKDRIVER",
|
||||
"runtimeVersion": "V8"
|
||||
}
|
||||
@ -0,0 +1,226 @@
|
||||
/***************
|
||||
* CONFIG
|
||||
***************/
|
||||
const CONFIG = {
|
||||
SPREADSHEET_ID: "17II0V8EsTL-9nHq2HgqhHcay-t-hhOaY5wNEzXHEsAM",
|
||||
SOURCE_SHEET_NAME: "WSMinMCA.csv", // tab name inside the spreadsheet
|
||||
EMAIL_COLUMN_INDEX_1BASED: 3, // Column C = 3 (1-based)
|
||||
SENDER: "nrasmussen@gainsight.com",
|
||||
SUBJECT: "DATA REQUEST",
|
||||
PROCESSED_LABEL: "DATA_REQUEST_PROCESSED",
|
||||
TEMP_SHEET_PREFIX: "DATA_REQUEST_", // new tab prefix
|
||||
MAX_THREADS_PER_RUN: 10 // safety throttle
|
||||
};
|
||||
|
||||
/**
|
||||
* Run once to create the time-driven trigger.
|
||||
* (There is no true "email received" trigger for Gmail in Apps Script.)
|
||||
*/
|
||||
function setupTrigger() {
|
||||
// Optional: delete existing triggers for this function to avoid duplicates
|
||||
ScriptApp.getProjectTriggers()
|
||||
.filter(t => t.getHandlerFunction() === "processDataRequests")
|
||||
.forEach(t => ScriptApp.deleteTrigger(t));
|
||||
|
||||
// Create a trigger every 5 minutes (adjust as desired)
|
||||
ScriptApp.newTrigger("processDataRequests")
|
||||
.timeBased()
|
||||
.everyMinutes(5)
|
||||
.create();
|
||||
}
|
||||
|
||||
/**
|
||||
* Main worker: finds unread "DATA REQUEST" messages from aubrey@ws.com,
|
||||
* extracts target email from body, filters rows, creates new tab, CSVs it,
|
||||
* replies with attachment, and labels thread as processed.
|
||||
*/
|
||||
function processDataRequests() {
|
||||
// const label = GmailApp.getUserLabelByName(CONFIG.PROCESSED_LABEL) || GmailApp.createLabel(CONFIG.PROCESSED_LABEL);
|
||||
|
||||
// // Search unread matching threads that are not already labeled processed
|
||||
// const query = [
|
||||
// `from:${CONFIG.SENDER}`,
|
||||
// `subject:"${CONFIG.SUBJECT}"`,
|
||||
// "is:unread",
|
||||
// `-label:${CONFIG.PROCESSED_LABEL}`
|
||||
// ].join(" ");
|
||||
|
||||
const threads = GmailApp.search(query, 0, CONFIG.MAX_THREADS_PER_RUN);
|
||||
Logger.log(threads);
|
||||
// if (!threads.length) return;
|
||||
|
||||
const ss = SpreadsheetApp.openById(CONFIG.SPREADSHEET_ID);
|
||||
const sourceSheet = ss.getSheetByName(CONFIG.SOURCE_SHEET_NAME);
|
||||
if (!sourceSheet) throw new Error(`Sheet tab not found: ${CONFIG.SOURCE_SHEET_NAME}`);
|
||||
|
||||
// Read all data once for efficiency
|
||||
const sourceValues = sourceSheet.getDataRange().getValues();
|
||||
if (sourceValues.length < 1) throw new Error(`No data found in ${CONFIG.SOURCE_SHEET_NAME}`);
|
||||
|
||||
const header = sourceValues[0];
|
||||
const emailColIdx0 = CONFIG.EMAIL_COLUMN_INDEX_1BASED - 1;
|
||||
|
||||
threads.forEach(thread => {
|
||||
const messages = thread.getMessages();
|
||||
|
||||
// Process only unread messages in the thread (usually just the newest)
|
||||
messages.forEach(message => {
|
||||
if (!message.isUnread()) return;
|
||||
|
||||
try {
|
||||
const plainBody = message.getPlainBody() || "";
|
||||
const targetEmail = extractFirstNonSenderEmail_(plainBody, CONFIG.SENDER);
|
||||
|
||||
if (!targetEmail) {
|
||||
message.reply(
|
||||
`I received your DATA REQUEST, but I couldn't find a target email address in the message body.\n\n` +
|
||||
`Please include an email like "someone@company.com" in the body and resend.`
|
||||
);
|
||||
message.markRead();
|
||||
return;
|
||||
}
|
||||
|
||||
const targetLower = targetEmail.toLowerCase();
|
||||
|
||||
// Filter rows where Column C exactly matches the extracted email (case-insensitive)
|
||||
const matches = sourceValues
|
||||
.slice(1) // skip header
|
||||
.filter(row => String(row[emailColIdx0] || "").trim().toLowerCase() === targetLower);
|
||||
|
||||
// Create a new tab with results
|
||||
const newSheetName = makeSafeSheetName_(
|
||||
`${CONFIG.TEMP_SHEET_PREFIX}${targetLower}_${Utilities.formatDate(new Date(), Session.getScriptTimeZone(), "yyyyMMdd_HHmmss")}`
|
||||
);
|
||||
|
||||
const resultSheet = ss.insertSheet(newSheetName);
|
||||
resultSheet.getRange(1, 1, 1, header.length).setValues([header]);
|
||||
|
||||
if (matches.length) {
|
||||
resultSheet.getRange(2, 1, matches.length, header.length).setValues(matches);
|
||||
}
|
||||
|
||||
// Convert result tab to CSV
|
||||
const csv = valuesToCsv_(resultSheet.getDataRange().getValues());
|
||||
const blob = Utilities.newBlob(csv, "text/csv", `${targetLower}.csv`);
|
||||
|
||||
// Reply with CSV attached (keeps threading)
|
||||
const body =
|
||||
`Attached is the CSV export for ${targetEmail}.\n\n` +
|
||||
`Rows matched (excluding header): ${matches.length}\n` +
|
||||
`Source tab: ${CONFIG.SOURCE_SHEET_NAME}\n` +
|
||||
`Result tab: ${newSheetName}\n`;
|
||||
|
||||
message.reply(body, { attachments: [blob] });
|
||||
|
||||
// Mark processed
|
||||
message.markRead();
|
||||
thread.addLabel(label);
|
||||
|
||||
// Optional cleanup: uncomment if you do NOT want to keep the result tab
|
||||
// ss.deleteSheet(resultSheet);
|
||||
|
||||
} catch (err) {
|
||||
// Fail gracefully: reply with error so the sender knows it didn't work
|
||||
message.reply(
|
||||
`I attempted to process this DATA REQUEST but hit an error:\n\n${err && err.stack ? err.stack : err}`
|
||||
);
|
||||
message.markRead();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function modifiedProcessDataRequests() {
|
||||
const ss = SpreadsheetApp.openById(CONFIG.SPREADSHEET_ID);
|
||||
const sourceSheet = ss.getSheetByName(CONFIG.SOURCE_SHEET_NAME);
|
||||
if (!sourceSheet) throw new Error(`Sheet tab not found: ${CONFIG.SOURCE_SHEET_NAME}`);
|
||||
const targetEmail = "khodge2@wsgc.com";
|
||||
const targetLower = targetEmail.toLowerCase();
|
||||
|
||||
// Read all data once for efficiency
|
||||
const sourceValues = sourceSheet.getDataRange().getValues();
|
||||
if (sourceValues.length < 1) throw new Error(`No data found in ${CONFIG.SOURCE_SHEET_NAME}`);
|
||||
|
||||
const header = sourceValues[0];
|
||||
Logger.log(header);
|
||||
const emailColIdx0 = CONFIG.EMAIL_COLUMN_INDEX_1BASED - 1;
|
||||
Logger.log(emailColIdx0);
|
||||
|
||||
// Filter rows where Column C exactly matches the extracted email (case-insensitive)
|
||||
const matches = sourceValues
|
||||
.slice(1) // skip header
|
||||
.filter(row => String(row[emailColIdx0] || "").trim().toLowerCase() === targetLower);
|
||||
|
||||
// Create a new tab with results
|
||||
const newSheetName = makeSafeSheetName_(
|
||||
`${CONFIG.TEMP_SHEET_PREFIX}${targetLower}_${Utilities.formatDate(new Date(), Session.getScriptTimeZone(), "yyyyMMdd_HHmmss")}`
|
||||
);
|
||||
|
||||
const resultSheet = ss.insertSheet(newSheetName);
|
||||
resultSheet.getRange(1, 1, 1, header.length).setValues([header]);
|
||||
|
||||
if (matches.length) {
|
||||
resultSheet.getRange(2, 1, matches.length, header.length).setValues(matches);
|
||||
}
|
||||
|
||||
// Convert result tab to CSV
|
||||
const csv = valuesToCsv_(resultSheet.getDataRange().getValues());
|
||||
const blob = Utilities.newBlob(csv, "text/csv", `${targetLower}.csv`);
|
||||
|
||||
// Reply with CSV attached (keeps threading)
|
||||
const body =
|
||||
`Attached is the CSV export for ${targetEmail}.\n\n` +
|
||||
`Rows matched (excluding header): ${matches.length}\n` +
|
||||
`Source tab: ${CONFIG.SOURCE_SHEET_NAME}\n` +
|
||||
`Result tab: ${newSheetName}\n`;
|
||||
|
||||
Logger.log(body);
|
||||
|
||||
// Optional cleanup: uncomment if you do NOT want to keep the result tab
|
||||
// ss.deleteSheet(resultSheet);
|
||||
}
|
||||
|
||||
/****************
|
||||
* HELPERS
|
||||
****************/
|
||||
|
||||
/**
|
||||
* Extracts the first email address found in the text that is NOT the sender email.
|
||||
*/
|
||||
function extractFirstNonSenderEmail_(text, senderEmail) {
|
||||
const re = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/ig;
|
||||
const found = text.match(re) || [];
|
||||
const senderLower = (senderEmail || "").toLowerCase();
|
||||
const first = found
|
||||
.map(e => e.trim())
|
||||
.find(e => e.toLowerCase() !== senderLower);
|
||||
return first || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a 2D array of values to RFC-ish CSV:
|
||||
* - Wrap fields in quotes if they contain comma, quote, or newline
|
||||
* - Double internal quotes
|
||||
*/
|
||||
function valuesToCsv_(values) {
|
||||
return values.map(row => row.map(cell => {
|
||||
let s = cell === null || cell === undefined ? "" : String(cell);
|
||||
// Normalize line endings inside cells
|
||||
s = s.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
||||
const needsQuotes = /[",\n]/.test(s);
|
||||
if (needsQuotes) s = `"${s.replace(/"/g, '""')}"`;
|
||||
return s;
|
||||
}).join(",")).join("\r\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Google Sheets tab names have limits; keep it safe.
|
||||
*/
|
||||
function makeSafeSheetName_(name) {
|
||||
// Remove characters Sheets doesn't like: [ ] : * ? / \
|
||||
let safe = name.replace(/[\[\]:*?/\\]/g, " ");
|
||||
safe = safe.replace(/\s+/g, " ").trim();
|
||||
// Max length 100
|
||||
if (safe.length > 100) safe = safe.slice(0, 100);
|
||||
return safe || "DATA_REQUEST_RESULT";
|
||||
}
|
||||
@ -0,0 +1,193 @@
|
||||
/***********************
|
||||
* PROPFIG (edit these)
|
||||
***********************/
|
||||
const PROPFIG = {
|
||||
spreadsheetId: "17II0V8EsTL-9nHq2HgqhHcay-t-hhOaY5wNEzXHEsAM",
|
||||
sheetId: 249063374, // the tab's sheetId (not gid string; usually same number)
|
||||
courseCol: 6, // Column C = 3
|
||||
headerRow: 1,
|
||||
startDataRow: 2,
|
||||
northpassBaseUrl: "https://api.northpass.com",
|
||||
cacheTtlSeconds: 6 * 60 * 60 // 6 hours
|
||||
};
|
||||
/**
|
||||
* One-time setup:
|
||||
* Run setNorthpassApiKey_() to store your API key securely in Script Properties.
|
||||
*/
|
||||
function setNorthpassApiKey() {
|
||||
const apiKey = "N5AQIwjIeeYDBSU9vTST37Nvg";
|
||||
PropertiesService.getScriptProperties().setProperty("NORTHPASS_API_KEY", apiKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entrypoint: reads course names from col C and writes core_competencies
|
||||
* into the first empty column (based on header row).
|
||||
*/
|
||||
function writeCoreCompetenciesFromCourses() {
|
||||
const ss = SpreadsheetApp.openById(PROPFIG.spreadsheetId);
|
||||
const sheet = getSheetById_(ss, PROPFIG.sheetId);
|
||||
|
||||
const apiKey = PropertiesService.getScriptProperties().getProperty("NORTHPASS_API_KEY");
|
||||
if (!apiKey) throw new Error("Missing NORTHPASS_API_KEY. Run setNorthpassApiKey_() first.");
|
||||
|
||||
// Determine output column: first empty cell in header row; if none, append.
|
||||
const lastCol = Math.max(sheet.getLastColumn(), 1);
|
||||
const headerValues = sheet.getRange(PROPFIG.headerRow, 1, 1, lastCol).getValues()[0];
|
||||
let outCol = headerValues.findIndex(v => String(v || "").trim() === "") + 1;
|
||||
if (outCol === 0) outCol = lastCol + 1;
|
||||
|
||||
// Write a header label (optional but useful)
|
||||
sheet.getRange(PROPFIG.headerRow, outCol).setValue("core_competencies");
|
||||
|
||||
// Read course column values
|
||||
const lastRow = sheet.getLastRow();
|
||||
if (lastRow < PROPFIG.startDataRow) return;
|
||||
|
||||
const numRows = lastRow - PROPFIG.startDataRow + 1;
|
||||
const courseValues = sheet.getRange(PROPFIG.startDataRow, PROPFIG.courseCol, numRows, 1).getValues();
|
||||
|
||||
// Prep caches: in-run memo + script cache
|
||||
const runMemo = {}; // { courseNameLower: { id, coreCompetenciesString } }
|
||||
const cache = CacheService.getScriptCache();
|
||||
|
||||
// Build output column values aligned to rows
|
||||
const output = new Array(numRows).fill([""]);
|
||||
|
||||
for (let i = 0; i < numRows; i++) {
|
||||
const rawCourse = String(courseValues[i][0] || "").trim();
|
||||
if (!rawCourse) continue;
|
||||
|
||||
const key = rawCourse.toLowerCase();
|
||||
|
||||
// 1) in-run memo
|
||||
if (runMemo[key]) {
|
||||
output[i] = [runMemo[key].coreCompetenciesString];
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2) CacheService (persists across runs for TTL)
|
||||
const cached = cache.get(cacheKey_(key));
|
||||
if (cached) {
|
||||
const parsed = JSON.parse(cached);
|
||||
runMemo[key] = parsed;
|
||||
output[i] = [parsed.coreCompetenciesString];
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3) Fetch from Northpass
|
||||
const courseId = fetchCourseIdByExactName_(apiKey, rawCourse);
|
||||
const coreCompetencies = fetchCoreCompetenciesByCourseId_(apiKey, courseId);
|
||||
|
||||
const coreStr = normalizeToSingleString_(coreCompetencies);
|
||||
|
||||
const record = { id: courseId, coreCompetenciesString: coreStr };
|
||||
runMemo[key] = record;
|
||||
cache.put(cacheKey_(key), JSON.stringify(record), PROPFIG.cacheTtlSeconds);
|
||||
|
||||
output[i] = [coreStr];
|
||||
}
|
||||
|
||||
// Write results in one batch
|
||||
sheet.getRange(PROPFIG.startDataRow, outCol, numRows, 1).setValues(output);
|
||||
}
|
||||
|
||||
/***********************
|
||||
* Northpass API helpers
|
||||
***********************/
|
||||
|
||||
/**
|
||||
* Gets course id by exact name using:
|
||||
* /v2/courses?filter[name][eq]=<courseName>
|
||||
* If no match is returned, falls back to:
|
||||
* /v2/courses?filter[name]=<courseName>
|
||||
*
|
||||
* Filtering is documented by Northpass. :contentReference[oaicite:3]{index=3}
|
||||
*/
|
||||
function fetchCourseIdByExactName_(apiKey, courseName) {
|
||||
const encoded = encodeURIComponent(courseName);
|
||||
|
||||
const urlsToTry = [
|
||||
`${PROPFIG.northpassBaseUrl}/v2/courses?filter[name][eq]=${encoded}`,
|
||||
`${PROPFIG.northpassBaseUrl}/v2/courses?filter[name]=${encoded}` // fallback
|
||||
];
|
||||
|
||||
for (const url of urlsToTry) {
|
||||
const json = northpassGetJson_(apiKey, url);
|
||||
|
||||
// Typical Northpass JSON: { data: [ { id, type, attributes... }, ... ] } :contentReference[oaicite:4]{index=4}
|
||||
const data = json && json.data;
|
||||
if (Array.isArray(data) && data.length > 0 && data[0].id) {
|
||||
return data[0].id;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`No course found for name "${courseName}" (tried eq + fallback).`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches course properties:
|
||||
* /v2/properties/courses/{id}
|
||||
* Returns: data.attributes.customized_properties.core_competencies :contentReference[oaicite:5]{index=5}
|
||||
*/
|
||||
function fetchCoreCompetenciesByCourseId_(apiKey, courseId) {
|
||||
const url = `${PROPFIG.northpassBaseUrl}/v2/properties/courses/${encodeURIComponent(courseId)}`;
|
||||
const json = northpassGetJson_(apiKey, url);
|
||||
|
||||
const core =
|
||||
json &&
|
||||
json.data &&
|
||||
json.data.attributes &&
|
||||
json.data.attributes.customized_properties &&
|
||||
json.data.attributes.customized_properties.core_competencies;
|
||||
|
||||
// Could be undefined/null/string/array/object depending on how it's stored
|
||||
return core ?? "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes an authenticated GET request to Northpass using X-Api-Key header.
|
||||
* Northpass API key auth is documented. :contentReference[oaicite:6]{index=6}
|
||||
*/
|
||||
function northpassGetJson_(apiKey, url) {
|
||||
const resp = UrlFetchApp.fetch(url, {
|
||||
method: "get",
|
||||
muteHttpExceptions: true,
|
||||
headers: {
|
||||
"Accept": "application/json",
|
||||
"X-Api-Key": apiKey
|
||||
}
|
||||
});
|
||||
|
||||
const code = resp.getResponseCode();
|
||||
const text = resp.getContentText();
|
||||
|
||||
if (code < 200 || code >= 300) {
|
||||
throw new Error(`Northpass API error ${code} for ${url}\n${text}`);
|
||||
}
|
||||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
/***********************
|
||||
* Sheet helpers
|
||||
***********************/
|
||||
function getSheetById_(spreadsheet, sheetId) {
|
||||
const sheets = spreadsheet.getSheets();
|
||||
for (const s of sheets) {
|
||||
if (s.getSheetId() === sheetId) return s;
|
||||
}
|
||||
throw new Error(`No sheet found with sheetId=${sheetId}`);
|
||||
}
|
||||
|
||||
/***********************
|
||||
* Formatting + caching
|
||||
***********************/
|
||||
function normalizeToSingleString_(value) {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (Array.isArray(value)) return value.map(v => String(v).trim()).filter(Boolean).join("; ");
|
||||
if (typeof value === "object") return JSON.stringify(value);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function cacheKey_(courseNameLower) {
|
||||
return `np_course_${courseNameLower}`;
|
||||
}
|
||||
Reference in New Issue
Block a user