Browse Source

🏰🐓 Updated with Glitch

master
Glitch (steady-sundial) 1 year ago
parent
commit
3b78294278
14 changed files with 781 additions and 256 deletions
  1. +6
    -0
      .sqlite_history
  2. +26
    -0
      fetch-missing-profiles.js
  3. +25
    -0
      gardener.js
  4. +52
    -0
      lib/check-links.js
  5. +23
    -0
      lib/db.js
  6. +72
    -0
      lib/models/site.js
  7. +53
    -0
      lib/representative-h-card.js
  8. +18
    -0
      one-off.js
  9. +9
    -9
      package.json
  10. +7
    -0
      public/style.css
  11. +76
    -128
      server.js
  12. +397
    -112
      shrinkwrap.yaml
  13. +16
    -6
      views/dashboard.hbs
  14. +1
    -1
      views/layouts/main.hbs

+ 6
- 0
.sqlite_history View File

@@ -0,0 +1,6 @@
select * from Sites where url like '%tantek.com%';
select * from Sites where url like '%tantek.com%';
select * from Sites where slug like '%❕?%';
select * from Sites where slug like '%â%';
select * from Sites where slug like '%â%';
select * from Sites where url like '%tantek.com%';

+ 26
- 0
fetch-missing-profiles.js View File

@@ -0,0 +1,26 @@
const DB = require('./lib/db');
const db = DB();

const Site = require('./lib/models/site');
const { saveProfile } = Site(db);

const fetchRepresentativeHCard = require('./lib/representative-h-card');

// i want to wait between requests, so let's use a recursive function
// that repeatedly sets a timeout to call itself!
function processSites(sites) {
const site = Array.prototype.pop.call(sites);
console.log(site.url, site.slug);
fetchRepresentativeHCard(site.url)
.then((card) => {
if(card) {
console.log(card);
return saveProfile(site, card);
}
});
setTimeout(processSites, 1000, sites);
}

db.all(`SELECT * FROM Sites WHERE profile IS NULL`, function(err, rows) {
processSites(rows);
});

+ 25
- 0
gardener.js View File

@@ -0,0 +1,25 @@
// we've gotta parse people's webpages
const checkSiteLinks = require('./lib/check-links');

const DB = require('./lib/db');
const db = DB();

const Sites = require('./lib/models/site');
const {addSite, getSite, getSiteBySlug, saveProfile, saveSiteCheckStatus} = Sites(db);

// i want to wait between requests, so let's use a recursive function
// that repeatedly sets a timeout to call itself!
function processSites(sites) {
const site = Array.prototype.pop.call(sites);
console.log(site.url, site.slug);
checkSiteLinks(site)
.then((status) => {
saveSiteCheckStatus(site, status);
console.log(site.url, site.slug, status);
setTimeout(processSites, 1000, sites);
});
}

db.all(`SELECT * FROM Sites`, function(err, rows) {
processSites(rows);
});

+ 52
- 0
lib/check-links.js View File

@@ -0,0 +1,52 @@
// we've gotta parse people's webpages
const URL = require('url').URL;
const jsdom = require("jsdom");
const { JSDOM } = jsdom;

module.exports = function (site) {
return new Promise((fulfill, reject) => {
const siteBase = 'https://' + process.env.MAIN_URL;
const expectedPaths = {
next: '/' + site.slug + '/next',
previous: '/' + site.slug + '/previous'
}
// FIXME: replace w/ request+cheerio, since microformat-node uses cheerio anyway?
JSDOM.fromURL(site.url)
.then(dom => {
const window = dom.window;
const doc = window.document;
let links = doc.querySelectorAll('a');
let candidatesByPath = Array.prototype.map.call(links, (a) => a.href)
.filter(href => href.startsWith(siteBase))
.map(href => new URL(href))
.reduce((by_path, candidate) => {
by_path[decodeURIComponent(candidate.pathname)] = candidate; return by_path;
}, {})
let found = {}
for (let linkType in expectedPaths) {
if (candidatesByPath[expectedPaths[linkType]]) {
found[linkType] = decodeURI(candidatesByPath[expectedPaths[linkType]].href);
delete candidatesByPath[expectedPaths[linkType]];
delete expectedPaths[linkType];
}
}
let result = {
found,
missing: expectedPaths,
mystery: candidatesByPath,
active: (Object.keys(found).length > 0)
};
// remove any empties
for (let key of ['found','missing','mystery']) {
if(Object.keys(result[key]).length === 0) {
delete result[key];
}
}
fulfill(result);
window.close();
})
.catch(err => {
fulfill({ status: 0, error: `Problem checking site link: ${err}` });
})
});
}

+ 23
- 0
lib/db.js View File

@@ -0,0 +1,23 @@
// init sqlite db
const fs = require('fs');
const sqlite3 = require('sqlite3').verbose();

module.exports = function() {
const dbFile = './.data/sqlite.db';
const exists = fs.existsSync(dbFile);
const db = new sqlite3.Database(dbFile);

// if ./.data/sqlite.db does not exist, create it
db.serialize(function(){
if (!exists) {
db.run('CREATE TABLE Sites (slug TEXT PRIMARY KEY, url TEXT, profile TEXT, active INTEGER, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP)');
console.log('New table Sites created!');
db.run('CREATE TABLE SiteChecks (slug TEXT, url TEXT, datetime TEXT DEFAULT CURRENT_TIMESTAMP, result TEXT)');
console.log('New table SiteChecks created!');
}
else {
console.log('Database "Sites" ready to go!');
}
});
return db;
}

+ 72
- 0
lib/models/site.js View File

@@ -0,0 +1,72 @@
// what if slugs were unicode?
const hashEmoji = require('hash-emoji-without-borders');

module.exports = function(db) {
const addSite = (url) => {
return new Promise((fulfill, reject) => {
let slug = hashEmoji(url, 3);
db.run(`INSERT INTO Sites (slug, url, active) VALUES ("${slug}","${url}", 1);`, function(err) {
if(err) {
reject({ message: `Couldn't add site ${url}`, error: err});
} else {
fulfill({url: url, slug: slug, active: 1});
}
})
});
};
const getSite = (url, create = false) => {
return new Promise((fulfill, reject) => {
// FIXME: sanitize url, reject if it isn't one.
db.get(`SELECT * from Sites where url="${url}"`, function(err, row) {
if (row == undefined) {
if (create) {
addSite(url)
.then((site) => fulfill(site))
.catch((err) => reject(err));
} else {
reject("Site not found.");
}
} else {
fulfill(row);
}
});
});
};
const getSiteBySlug = (slug) => {
return new Promise((fulfill, reject) => {
// FIXME: sanitize slug
db.get(`SELECT * from Sites where slug="${slug}"`, function(err, row) {
if (!err) {
fulfill(row);
} else {
reject(err);
}
});
});
};
const saveProfile = (site, profile) => {
return new Promise((fulfill, reject) => {
db.run(`UPDATE Sites SET PROFILE = ? WHERE SLUG = ?;`, JSON.stringify(profile), site.slug);
fulfill();
});
};
const saveSiteCheckStatus = (site, status) => {
return new Promise((fulfill, reject) => {
db.run(`INSERT INTO SiteChecks (slug, url, result) VALUES (?, ?, ?)`, site.slug,site.url, JSON.stringify(status));
db.run(`UPDATE Sites SET ACTIVE = ? WHERE SLUG = ?;`, status.active, site.slug);
fulfill();
});
};
return {
addSite,
getSite,
getSiteBySlug,
saveProfile,
saveSiteCheckStatus
};
}

+ 53
- 0
lib/representative-h-card.js View File

@@ -0,0 +1,53 @@
const Request = require('request');
const MF2 = require('microformat-node');

function hasPropValue(mfObj, prop, value) {
return ('properties' in mfObj) && (prop in mfObj['properties']);// && (mfObj['properties'][prop].indexOf(value) !== -1);
}

function hasRelValue(rels, rel, value) {
return (rel in rels) && (rels[rel].indexOf(value) !== -1);
}

module.exports = function (url) {
return new Promise((fulfill, reject) => {
Request(url, (error, res, html) => {
if (!error && res.statusCode == 200) {
// a pass at implementing http://microformats.org/wiki/representative-h-card-parsing
const parsedMf2 = MF2.get({
'baseUrl': url,
'html': html,
'filters': ['h-card']
});
const cards = parsedMf2.items;
// if no cards, no card!
if(cards.length == 0){
fulfill(false);
}
// if a card has a uid and url of the current url, use it.
for( let card of cards ){
if(hasPropValue(card, 'uid', url) && hasPropValue(card, 'url', url)) {
fulfill(card);
}
}
// if a card has a url that is also a rel=me on the page, use it.
for( let card of cards ){
if(hasRelValue(parsedMf2.rels, 'me', url)){
fulfill(card);
}
}
// last chance - if there is only 1 card and url matches current url, use it.
if((cards.length == 1) && hasPropValue(cards[0], 'url', url)) {
fulfill(cards[0]);
}
// otherwise, no deal!
fulfill(false);
} else {
reject(error);
}
});
});
}

+ 18
- 0
one-off.js View File

@@ -0,0 +1,18 @@
const DB = require('./lib/db');
const db = DB();

const hashEmoji = require('hash-emoji-without-borders');

const addSite = (url, slug) => {
return new Promise((fulfill, reject) => {
db.run(`INSERT INTO Sites (slug, url, active) VALUES ("${slug}","${url}", 1);`, function(err) {
if(err) {
reject({ message: `Couldn't add site ${url}`, error: err});
} else {
fulfill({url: url, slug: slug, active: 1});
}
})
});
};

addSite('http://tantek.com/', '📊');

+ 9
- 9
package.json View File

@@ -1,27 +1,27 @@
{
"//1": "describes your app and its dependencies",
"//2": "https://docs.npmjs.com/files/package.json",
"//3": "updating this file will download and update your packages",
"name": "hello-express",
"name": "steady-sundial",
"version": "0.0.1",
"description": "A simple Node app built on Express, instantly up and running.",
"description": "A Node app built on Express, for running an IndieWeb Webring.",
"main": "server.js",
"scripts": {
"start": "node server.js"
},
"dependencies": {
"express": "^4.16.3",
"sqlite3": "^4.0.1",
"cheerio": "^0.22.0",
"cookie-session": "^1.3.2",
"express": "^4.16.4",
"express-handlebars": "^3.0.0",
"hash-emoji-without-borders": "git+https://github.com/martymcguire/hash-emoji-without-borders#master",
"indieauth-authentication": "^0.0.3",
"hash-emoji-without-borders": "git+https://github.com/martymcguire/hash-emoji-without-borders#master"
"microformat-node": "^2.0.1",
"request": "^2.88.0",
"sqlite3": "^4.0.4"
},
"engines": {
"node": "8.x"
},
"repository": {
"url": "https://glitch.com/edit/#!/hello-express"
"url": "https://glitch.com/edit/#!/steady-sundial"
},
"license": "MIT",
"keywords": [

+ 7
- 0
public/style.css View File

@@ -69,3 +69,10 @@ footer {
padding-top: 25px;
border-top: 1px solid lightgrey;
}

.profile {
display: grid;
}
.profile > img {
max-width: 100%;
}

+ 76
- 128
server.js View File

@@ -34,16 +34,23 @@ app.engine('hbs', hbs.engine);
app.set('views', __dirname + '/views');
app.set('view engine', 'hbs');

// what if slugs were unicode?
const hashEmoji = require('hash-emoji-without-borders');

// we've gotta parse people's webpages
const jsdom = require("jsdom");
const { JSDOM } = jsdom;
const checkSiteLinks = require('./lib/check-links');

// and get their h-cards for profile data
const fetchRepresentativeHCard = require('./lib/representative-h-card');

const DB = require('./lib/db');
const db = DB();

const Sites = require('./lib/models/site');
const {addSite, getSite, getSiteBySlug, saveProfile, saveSiteCheckStatus} = Sites(db);

// middleware to always set up a context to pass to templates
app.use(function(req, res, next){
res.locals.context = {};
res.locals.context = {
'projectUrl': 'https://glitch.com/~' + process.env.PROJECT_NAME
};
if(req.session.user) {
res.locals.context['user'] = req.session.user
}
@@ -59,103 +66,17 @@ function requireLogin(request, response, next) {
// http://expressjs.com/en/starter/static-files.html
app.use(express.static('public'));

// init sqlite db
var fs = require('fs');
var dbFile = './.data/sqlite.db';
var exists = fs.existsSync(dbFile);
var sqlite3 = require('sqlite3').verbose();
var db = new sqlite3.Database(dbFile);

// if ./.data/sqlite.db does not exist, create it
db.serialize(function(){
if (!exists) {
db.run('CREATE TABLE Sites (slug TEXT PRIMARY KEY, url TEXT, active INTEGER, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP)');
console.log('New table Sites created!');
db.run('CREATE TABLE SiteChecks (slug TEXT, url TEXT, datetime TEXT DEFAULT CURRENT_TIMESTAMP, result TEXT)');
console.log('New table SiteChecks created!');
}
else {
console.log('Database "Sites" ready to go!');
function profileFromHCard(card) {
let profile = {};
if ( ! 'properties' in card ) {
return profile;
}
});

function addSite(url) {
return new Promise((fulfill, reject) => {
let slug = hashEmoji(url, 3);
db.run(`INSERT INTO Sites (slug, url, active) VALUES ("${slug}","${url}", 1);`, function(err) {
if(err) {
reject({ message: `Couldn't add site ${url}`, error: err});
} else {
fulfill({url: url, slug: slug, active: 1});
}
})
});
}

function getSite(url, create = false) {
return new Promise((fulfill, reject) => {
db.get(`SELECT * from Sites where url="${url}"`, function(err, row) {
if ((row == undefined) && create) {
addSite(url)
.then((site) => fulfill(site))
.catch((err) => reject(err))
} else {
fulfill(row);
}
});
});
}

function checkSiteLinks(site) {
return new Promise((fulfill, reject) => {
const siteBase = 'https://' + process.env.MAIN_URL;
const expectedPaths = {
next: '/' + site.slug + '/next',
previous: '/' + site.slug + '/previous'
["name","note","photo"].forEach((prop) => {
if (prop in card.properties) {
profile[prop] = card.properties[prop][0];
}
JSDOM.fromURL(site.url)
.then(dom => {
const window = dom.window;
const doc = window.document;
let links = doc.querySelectorAll('a');
let candidatesByPath = Array.prototype.map.call(links, (a) => a.href)
.filter(href => href.startsWith(siteBase))
.map(href => new URL(href))
.reduce((by_path, candidate) => {
by_path[decodeURIComponent(candidate.pathname)] = candidate; return by_path;
}, {})
let found = {}
for (let linkType in expectedPaths) {
if (candidatesByPath[expectedPaths[linkType]]) {
found[linkType] = decodeURI(candidatesByPath[expectedPaths[linkType]].href);
delete candidatesByPath[expectedPaths[linkType]];
delete expectedPaths[linkType];
}
}
let result = {
found,
missing: expectedPaths,
mystery: candidatesByPath,
active: (Object.keys(found).length > 0)
};
// remove any empties
for (let key of ['found','missing','mystery']) {
if(Object.keys(result[key]).length === 0) {
delete result[key];
}
}
fulfill(result);
window.close();
})
.catch(err => {
fulfill({ status: 0, error: `Problem checking site link: ${err}` });
})
});
}

function saveSiteCheckStatus(site, status) {
db.run(`INSERT INTO SiteChecks (slug, url, result) VALUES (?, ?, ?)`, site.slug,site.url, JSON.stringify(status));
db.run(`UPDATE Sites SET ACTIVE = ? WHERE SLUG = ?;`, status.active, site.slug);
return profile;
}

// http://expressjs.com/en/starter/basic-routing.html
@@ -166,17 +87,40 @@ app.get("/", function (request, response) {
app.get('/dashboard', requireLogin, function(request, response) {
getSite(request.session.user.me, true) // find site in the db or add it
.then((site) => {
db.all(`SELECT * FROM SiteChecks WHERE url="${site.url}" ORDER BY datetime DESC LIMIT 3`, function(err, rows) {
let parsed_rows = Array.prototype.map.call(rows, row => {
row.result = JSON.parse(row.result);
return row;
return new Promise((fulfill, reject) => {
db.all(`SELECT * FROM SiteChecks WHERE url="${site.url}" ORDER BY datetime DESC LIMIT 3`, function(err, rows) {
let parsed_rows = Array.prototype.map.call(rows, row => {
row.result = JSON.parse(row.result);
return row;
});
if(site.profile) {
site.profile = profileFromHCard(JSON.parse(site.profile));
}
const u = new URL(site.url);
site.cute = u.hostname + u.pathname.replace(/\/$/,'')
response.locals.context['site'] = site;
response.locals.context['hostname'] = process.env.MAIN_URL;
response.locals.context['checks'] = rows;
fulfill();
});
});
response.locals.context['site'] = site;
response.locals.context['hostname'] = process.env.MAIN_URL;
response.locals.context['checks'] = rows;
})
.then(() => {
response.render('dashboard', response.locals.context);
});
});
});

app.post('/check-profile', requireLogin, function(request, response) {
getSite(request.session.user.me)
.then((site) => {
fetchRepresentativeHCard(site.url)
.then((card) => {
if(card) {
return saveProfile(site, card);
}
})
})
.then(() => { response.redirect('/dashboard'); });
});

app.post('/check-links', requireLogin, function(request, response) {
@@ -184,41 +128,45 @@ app.post('/check-links', requireLogin, function(request, response) {
.then((site) => {
if(site) {
checkSiteLinks(site)
.then((status) => {
saveSiteCheckStatus(site, status);
response.redirect('/dashboard');
});
.then((status) => saveSiteCheckStatus(site, status))
.then(() => { response.redirect('/dashboard'); });
}
});
});

app.get('/:slug/next', function(request, response) {
db.get(`SELECT * from Sites where slug="${request.params.slug}"`, function(err, row) {
db.get(`SELECT * from Sites where active = 1 AND slug != "${request.params.slug}" ORDER BY RANDOM() LIMIT 1`, function(err, row) {
response.redirect(row['url']);
getSiteBySlug(request.params.slug)
.then((site) => {
db.get(`SELECT * from Sites where active = 1 AND slug != "${site.slug}" ORDER BY RANDOM() LIMIT 1`, function(err, row) {
response.redirect(row['url']);
});
})
.catch(() => {
console.log('Tried to next unknown slug ', request.params.slug);
response.redirect('/');
});
});
});

app.get('/:slug/previous', function(request, response) {
db.get(`SELECT * from Sites where slug="${request.params.slug}"`, function(err, row) {
db.get(`SELECT * from Sites where active = 1 AND slug != "${request.params.slug}" ORDER BY RANDOM() LIMIT 1`, function(err, row) {
response.redirect(row['url']);
getSiteBySlug(request.params.slug)
.then((site) => {
db.get(`SELECT * from Sites where active = 1 AND slug != "${site.slug}" ORDER BY RANDOM() LIMIT 1`, function(err, row) {
response.redirect(row['url']);
});
})
.catch(() => {
console.log('Tried to previous unknown slug ', request.params.slug);
response.redirect('/');
});
});
});

app.get('/:slug', function(request, response) {
db.get(`SELECT * from Sites where slug="${request.params.slug}"`, function(err, row) {
if(row) {
response.redirect(row['url']);
} else {
response.redirect('/');
}
});
getSiteBySlug(request.params.slug)
.then((site) => { response.redirect(site['url']); })
.catch(() => { response.redirect('/'); });
});

// listen for requests :)
var listener = app.listen(process.env.PORT, function () {
console.log('Your app is listening on port ' + listener.address().port);
});
});getSiteBySlug

+ 397
- 112
shrinkwrap.yaml
File diff suppressed because it is too large
View File


+ 16
- 6
views/dashboard.hbs View File

@@ -1,7 +1,19 @@
<h3>
Hello, {{ site.url }} ({{site.slug}})!
Hello, {{ site.url }} ({{ site.slug }})!
</h3>

<form action="/check-profile" method="POST">
{{#if site.profile }}
<section class="profile">
{{#if site.profile.photo }}<img src="{{ site.profile.photo }}" />{{/if}}
<a href="{{ site.url }}">{{ site.cute }}</a>
{{#if site.profile.name }}<div class="name">{{ site.profile.name }}</div>{{/if}}
{{#if site.profile.note }}<div class="note">{{ site.profile.note }}</div>{{/if }}
</section>
{{/if}}
<input type="submit" value="Fetch updated profile."/>
</form>

<div>
Your site is currently: {{#unless site.active }}NOT {{/unless}} ACTIVE.
</div>
@@ -12,9 +24,9 @@

<div>
<textarea rows="6" cols="60">
<a href="https://{{ hostname }}/{{site.slug}}/previous">←</a>
<a href="https://{{ hostname }}/{{ site.slug }}/previous">←</a>
An IndieWeb Webring 🕸💍
<a href="https://{{ hostname }}/{{site.slug}}/next">→</a>
<a href="https://{{ hostname }}/{{ site.slug }}/next">→</a>
</textarea>
</div>

@@ -22,6 +34,4 @@ An IndieWeb Webring 🕸💍
<input type="submit" value="Check links now!"/>
</form>

{{#if checks }}
{{> checks }}
{{/if}}
{{#if checks }}{{> checks }}{{/if}}

+ 1
- 1
views/layouts/main.hbs View File

@@ -36,7 +36,7 @@

<footer>
<p>
Made with <a href="https://glitch.com">Glitch</a> and ❤️ at <a href="https://indieweb.org/2018">IndieWeb Summit 2018</a> by <a href="https://martymcgui.re/">schmarty</a>.
Made with <a href="https://glitch.com">Glitch 🐟🐟</a> (<a href="{{ projectUrl }}">Check out the Project</a>) and ❤️ at <a href="https://indieweb.org/2018">IndieWeb Summit 2018</a> by <a href="https://martymcgui.re/">schmarty</a>.
</p>
<p>
Having trouble? Find schmarty in the <a href="https://indieweb.org/discuss">IndieWeb chat</a>!

Loading…
Cancel
Save