Browse Source

New version, which is faster :)

master
Sebastian 3 years ago
parent
commit
597491af02
Signed by: imo GPG Key ID: AFA10CBFE4391C0C
  1. 2
      .gitignore
  2. 1
      .luacheckrc
  3. 31
      LICENSE
  4. 66
      README.md
  5. 24
      emoji-downloader-1.0-0.rockspec
  6. 116
      emoji-downloader.lua
  7. 122
      src/emoji_downloader.lua

2
.gitignore

@ -0,0 +1,2 @@
/lua
/lua_modules

1
.luacheckrc

@ -1,2 +1 @@
max_comment_line_length = false
std = 'max'

31
LICENSE

@ -1,7 +1,24 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Everyone is permitted to copy and distribute verbatim or modified copies of this license document, and changing it is allowed as long as the name is changed.
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.
In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We make this dedication for the benefit
of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <http://unlicense.org/>

66
README.md

@ -9,38 +9,59 @@ Inspired by [emoji-stealer](https://github.com/mirro-chan/emoji-stealer)
## Dependencies
* lua (>=5.1)
* [lua-requests](https://github.com/JakobGreen/lua-requests)
* [lpath](https://github.com/starwing/lpath)
* [copas](https://github.com/keplerproject/copas)
* [luasec](https://github.com/brunoos/luasec)
* [lua-rapidjson](https://github.com/xpol/lua-rapidjson)
* [argparse](https://github.com/mpeterv/argparse)
"Uhhh, why so much more dependencies?" you might think, so let me explain: The previous version depended on [lua-requests](https://github.com/JakobGreen/lua-requests), which has 5 dependencies on it's own, of which two were not even needed for this script. So we're at the same dependency level as before, but this time all dependencies are used. I could remove the argparse dependenciy and write my own short version, but...laziness. ;)
## Install
The easiest way to install this script is to install it directly with [luarocks](https://luarocks.org/#quick-start) like this:
### Method one
```
luarocks install https://raw.githubusercontent.com/imolein/emoji-downloader/master/emoji-downloader-scm-0.rockspec
```
* Download and install using the rockspec directly from the repository: `sudo luarocks install https://git.kokolor.es/imo/emoji-downloader/raw/branch/master/emoji-downloader-1.0-0.rockspec`
* The script should be installed in `/usr/local/bin/`, so you can use it like this: `emoji-downloader domain.tld`
* You can also use the `--local` option, if you don't want to install it global. If you do so, the script is install in `~/.luarocks/` and you have to use the full path to use it
But you can also clone this git, install the dependencies listed above using [luarocks](https://luarocks.org/#quick-start) and run the script directly.
### Method two
* Clone this git
* Change into the cloned folder
* Install dependencies:
```
luarocks install lua-requests
luarocks init
luarocks init --reset
luarocks build --only-deps
```
* After the installation is complete you can run it like this: `./lua emoji-downloader.lua domain.tld`
## Usage
If installed directly with **luarocks** use it like this:
```
emoji-downloader: emoji-downloader [OPTIONS] URL
Download custom emoji's from a Pleroma or Mastodon instance.
Useable options:
-d FOLDER Define the folder where the downloaded emoji's are stored (default: /tmp)
-ap API PATH Define the custom emoji api path (defaul: /api/v1/custom_emojis)
-et Generates the emoji.txt which is needed for Pleroma
-h Shows this message
-v Verbose - shows a message per downloaded emoji
Usage: emoji-downloader [-a <apipath>] [-c <concurrent>] [-d <dir>]
[-v] [-h] <url>
Download emojis from mastodon or pleroma instances
Arguments:
url The URL of the instance, from where you want to download the emoji's
Options:
-a <apipath>, Define the custom emoji api path (default: /api/v1/custom_emojis)
--apipath <apipath>
-c <concurrent>,
--concurrent <concurrent>
Concurrent downloads
-d <dir>, Define the directory where the downloaded emoji's are stored (default: /tmp/)
--dir <dir>
-v, --verbose Shows a message per downloaded emoji
-h, --help Show this help message and exit.
```
If you cloned this repository and installed only the dependencies with luarocks, you can use it the same way as described above, but with .lua file extension at the end:
@ -49,6 +70,17 @@ If you cloned this repository and installed only the dependencies with luarocks,
lua emoji-downloader.lua "https://example.com"
```
## Changelog
### 1.0-0 (2019-02-20)
* reworte most of the stuff
* concurrent download of emoji's (default 5 at once)
* Changed license from WTFPL to Unlicense
### scm-0 (2018-10-19)
* first version
## Works with

24
emoji-downloader-scm-0.rockspec → emoji-downloader-1.0-0.rockspec

@ -1,8 +1,9 @@
rockspec_format = '3.0'
package = 'emoji-downloader'
version = 'scm-0'
version = '1.0-0'
source = {
url = "git://github.com/imolein/emoji-downloader",
branch = "master"
tag = "v1.0-0"
}
description = {
summary = 'Download Pleroma or Mastodon custom emojis',
@ -11,20 +12,21 @@ description = {
I want them too."? Then just use this script to download
them.
]],
homepage = 'http://olivinelabs.com/busted/',
license = 'WTPFL <http://www.wtfpl.net/>'
homepage = 'https://git.kokolor.es/imo/emoji-downloader',
license = 'Unlicense'
}
dependencies = {
'lua >= 5.1',
'lua-requests',
'lpath'
'lua >= 5.3',
'lpath',
'copas',
'rapidjson',
'argparse',
'luasocket',
'luasec'
}
build = {
type = 'builtin',
modules = {
['emoji_downloader'] = 'src/emoji_downloader.lua'
},
type = 'none',
install = {
bin = {
['emoji-downloader'] = 'emoji-downloader.lua'

116
emoji-downloader.lua

@ -1,11 +1,111 @@
#!/usr/bin/env lua
--
-- Copyright © 2018 Sebastian Huebner
-- This work is free. You can redistribute it and/or modify it under the
-- terms of the Do What The Fuck You Want To Public License, Version 2,
-- as published by Sam Hocevar. See the LICENSE file or http://www.wtfpl.net/
-- for more details.
-- Download emojis from mastodon or pleroma instances
--
local copas = require('copas')
local http = require('copas.http')
local limit = require('copas.limit')
local url_parser = require('socket.url').parse
local ltn12 = require('ltn12') --ltn12
local json = require('rapidjson').decode --json
local argparse = require('argparse')
local exists = require('path.fs').exists
local mkdir = require('path.fs').makedirs
http.SSLPROTOCOL = 'tlsv1_2'
http.USERAGENT = 'emoji-downloader/scm-1 (https://codeberg.org/imo/emoji-downloader)'
-- Parse given arguments
local function parse_args()
local parser = argparse({
name = 'emoji-downloader',
description = 'Download emojis from mastodon or pleroma instances'
})
local opts
parser:option('-a --apipath', 'Define the custom emoji api path', '/api/v1/custom_emojis')
parser:option('-c --concurrent', 'Concurrent downloads', 5)
:convert(tonumber)
parser:option('-d --dir', 'Define the directory where the downloaded emoji\'s are stored', '/tmp/')
parser:flag('-v --verbose', 'Shows a message per downloaded emoji')
parser:argument('url', 'The URL of the instance, from where you want to download the emoji\'s')
:args(1)
:action(function(dt, index, arg)
local parsed = url_parser(arg)
local url = parsed.scheme and arg or 'https://' .. arg
dt[index] = ( url:sub(-1) == '/' and url:sub(1, -2) ) or url
end)
opts = parser:parse()
opts.dir = opts.dir .. opts.url:match('^https?://([%w%.]+%w+)/?$') .. '/'
return opts
end
-- Download custom emoji list from given url
local function download_list(url)
local jresp, resp, err
copas.addthread(function(_url)
resp, err = http.request(_url)
end, url)
copas.loop()
assert(resp, ('Download of emoji list failed with: %s'):format(err))
jresp, err = json(resp)
assert(jresp, ('Decoding of JSON object failed: %s'):format(err))
return jresp
end
-- Downloads and saves the emoji
-- Works only inside copas dispatcher
local function download_file(url, filename, folder, verbose)
local _, code = http.request({
url = url,
method = 'GET',
sink = ltn12.sink.file(io.open(folder .. filename, 'wb')),
redirect = 'all'
})
if verbose and code == 200 then
io.write(('Downloaded emoji: %s\n'):format(filename))
end
end
-- main function
local function main()
local opts = parse_args()
io.write('emoji-downloader started...\n')
if not exists(opts.dir) then
assert(mkdir(opts.dir), 'Couldn\'t create directory!')
io.write(('Created folder %s\n'):format(opts.dir))
end
io.write(('Download custom emoji list from %s\n'):format(opts.url))
local emoji_list = download_list(opts.url .. opts.apipath)
io.write(('Start downloading %d emojis to %s\n'):format(#emoji_list, opts.dir))
copas.addthread(function(list, _opts)
local query = limit.new(_opts.concurrent)
for _, e in ipairs(list) do
local filename = e.shortcode .. e.url:match('https?://.+/.*(%.%w+)[%?%d]*$')
query:addthread(download_file, e.url, filename, _opts.dir, _opts.verbose)
end
query:wait()
end, emoji_list, opts)
package.path = './src/?.lua;' .. package.path
copas.loop()
io.write(('Done! You can find the emoji\'s in %s\n'):format(opts.dir))
end
require('emoji_downloader')(arg)
main()

122
src/emoji_downloader.lua

@ -1,122 +0,0 @@
#!/usr/bin/env lua
--
-- Download emojis from mastodon or pleroma instances
--
-- Copyright © 2018 Sebastian Huebner
-- This work is free. You can redistribute it and/or modify it under the
-- terms of the Do What The Fuck You Want To Public License, Version 2,
-- as published by Sam Hocevar. See the LICENSE file or http://www.wtfpl.net/
-- for more details.
local requests = require('requests')
local exists = require('path.fs').exists
local mkdir = require('path.fs').makedirs
local emoji_downloader = {}
-- prints the help message
local function show_help()
io.write('emoji-downloader: emoji-downloader [OPTIONS] URL\n' ..
'Download custom emoji\'s from a Pleroma or Mastodon instance.\n\n' ..
'Useable options:\n' ..
' -d FOLDER Define the folder where the downloaded emoji\'s are stored (default: /tmp)\n' ..
' -ap API PATH Define the custom emoji api path (defaul: /api/v1/custom_emojis)\n' ..
' -et Generates the emoji.txt which is needed for Pleroma\n' ..
' -h Shows this message\n' ..
' -v Verbose - shows a message per downloaded emoji\n\n')
os.exit(0)
end
-- returns the url if valid, if not it return nil
local function validate_fqdn(url)
return url:match('^https?://[%w%.]+%w+/?$')
end
-- check and format the options
local function finalize_opts(opts)
assert(validate_fqdn(opts.url), 'URL format incorrect!')
opts.url = ( opts.url:sub(-1) == '/' and opts.url:sub(1, -2) ) or opts.url
opts.dest = (( opts.dest:sub(-1) ~= '/' and opts.dest .. '/' ) or opts.dest ) ..
opts.url:match('^https?://([%w%.]+%w+)/?$') .. '/'
return opts
end
-- parse given arguments
local function parse_args(args)
if #args == 0 then show_help() end
local opts = {
verbose = false,
create_emoji_txt = false,
url = args[#args],
ce_api_path = '/api/v1/custom_emojis',
dest = '/tmp/'
}
for i, a in ipairs(args) do
if a == '-h' then
show_help()
elseif a == '-d' then
opts.dest = args[i + 1]
elseif a == '-v' then
opts.verbose = true
elseif a == '-et' then
opts.create_emoji_txt = true
elseif a == '-ap' then
opts.ce_api_path = args[i + 1]
end
end
return finalize_opts(opts)
end
-- downloads and saves the emoji
local function download_file(url, filename, folder)
local resp, err = requests.get(url)
if not resp then return nil, err end
local file = io.open(folder .. filename, 'wb')
file:write(resp.text)
file:close()
return true
end
-- main function
function emoji_downloader.main(args)
local opts = parse_args(args)
if not exists(opts.dest) then assert(mkdir(opts.dest), 'Couldn\'t create directory!') end
-- get the list of emoji's
local emoji_list = assert(requests.get(opts.url .. opts.ce_api_path).json())
-- iterate over the list and download the files to the given download destination
for _, e in ipairs(emoji_list) do
local filename = e.shortcode .. e.url:match('https?://.+/.*(%.%w+)[%?%d]*$')
local success, err = download_file(e.url, filename, opts.dest)
if success then
if opts.verbose then
io.write(('Downloaded emoji: %s\n'):format(e.shortcode))
end
if opts.create_emoji_txt then
local file = io.open(opts.dest .. 'emoji.txt', 'a')
if file then
file:write(('%s, /emoji/custom/%s\n'):format(e.shortcode, filename))
file:close()
end
end
else
io.write(('The following error occured during the download of %s: %s\n'):format(e.shortcode, err))
end
end
io.write(('Look into folder %s to find the downloaded emoji\'s!\n'):format(opts.dest))
end
return emoji_downloader.main