build: fix doc generation

also moved MD5 to koreader-base/ffi
pull/1843/head
Qingping Hou 8 years ago
parent 042d94c3ef
commit 69f9ebd528

@ -1,16 +1,41 @@
#!/usr/bin/env bash #!/usr/bin/env bash
CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${CI_DIR}/common.sh"
set +e
make coverage make coverage
cd koreader-*/koreader && luajit $(which luacov-coveralls) -v pushd koreader-*/koreader
luajit $(which luacov-coveralls) -v
# get deploy key for doc repo popd
openssl aes-256-cbc -K $encrypted_dc71a4fb8382_key -iv $encrypted_dc71a4fb8382_iv \
-in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d if [ $TRAVIS_BRANCH == 'master' ]; then
ssh-add ~/.ssh/koreader_doc
git clone git@github.com:koreader/doc.git travis_retry luarocks --local install ldoc
# push doc update # get deploy key for doc repo
make doc openssl aes-256-cbc -k $doc_build_secret -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d
cp -r doc/html/* doc/ chmod 600 ~/.ssh/koreader_doc # make agent happy
pushd doc eval "$(ssh-agent)" > /dev/null
git add . ssh-add ~/.ssh/koreader_doc > /dev/null
git push origin gh-pages echo -e "\n${ANSI_GREEN}Check out koreader/doc for update."
git clone git@github.com:koreader/doc.git koreader_doc
# push doc update
pushd doc
luajit $(which ldoc) . 2> /dev/null
if [ ! -d html ]; then
echo "Failed to generate documents..."
exit 1
fi
popd
cp -r doc/html/* koreader_doc/
pushd koreader_doc
echo -e "\n${ANSI_GREEN}Pusing document update..."
git -c user.name="KOReader build bot" -c user.email="non-reply@koreader.rocks" \
commit -a --amend -m 'Automated documentation build from travis-ci.'
git push -f --quiet origin gh-pages > /dev/null
echo -e "\n${ANSI_GREEN}Document update pushed."
fi

@ -1,3 +1,6 @@
set -e
set -o pipefail
ANSI_RED="\033[31;1m" ANSI_RED="\033[31;1m"
ANSI_GREEN="\033[32;1m" ANSI_GREEN="\033[32;1m"
ANSI_RESET="\033[0m" ANSI_RESET="\033[0m"
@ -6,6 +9,8 @@ ANSI_CLEAR="\033[0K"
travis_retry() { travis_retry() {
local result=0 local result=0
local count=1 local count=1
set +e
while [ $count -le 3 ]; do while [ $count -le 3 ]; do
[ $result -ne 0 ] && { [ $result -ne 0 ] && {
echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2
@ -21,6 +26,7 @@ travis_retry() {
echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2 echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2
} }
set -e
return $result return $result
} }

@ -15,6 +15,7 @@ mkdir $HOME/.luarocks
cp ${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua $HOME/.luarocks/config.lua cp ${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua $HOME/.luarocks/config.lua
echo "wrap_bin_scripts = false" >> $HOME/.luarocks/config.lua echo "wrap_bin_scripts = false" >> $HOME/.luarocks/config.lua
travis_retry luarocks --local install luafilesystem travis_retry luarocks --local install luafilesystem
# for verbose_print module
travis_retry luarocks --local install ansicolors travis_retry luarocks --local install ansicolors
travis_retry luarocks --local install busted 2.0.rc11-0 travis_retry luarocks --local install busted 2.0.rc11-0
#- travis_retry luarocks --local install busted 1.11.1-1 #- travis_retry luarocks --local install busted 1.11.1-1
@ -25,4 +26,3 @@ travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linu
travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev
travis_retry luarocks --local install luacheck travis_retry luarocks --local install luacheck
travis_retry luarocks --local install lanes # for parallel luacheck travis_retry luarocks --local install lanes # for parallel luacheck
travis_retry luarocks --local install ldoc

Binary file not shown.

@ -6,5 +6,6 @@ source "${CI_DIR}/common.sh"
travis_retry make fetchthirdparty travis_retry make fetchthirdparty
make all make all
travis_retry make testfront travis_retry make testfront
set +o pipefail
luajit $(which luacheck) --no-color -q frontend | tee ./luacheck.out luajit $(which luacheck) --no-color -q frontend | tee ./luacheck.out
test $(grep Total ./luacheck.out | awk '{print $2}') -le 63 test $(grep Total ./luacheck.out | awk '{print $2}') -le 63

2
.gitignore vendored

@ -33,7 +33,7 @@ i18n
koreader-android-arm-linux-androideabi koreader-android-arm-linux-androideabi
koreader-kindle-legacy-arm-kindle-linux-gnueabi koreader-kindle-legacy-arm-kindle-linux-gnueabi
koreader-kindle-arm-linux-gnueabi koreader-kindle-arm-linux-gnueabi
koreader-kobo-arm-linux-gnueabihf koreader-kobo-arm-linux-gnueabihf*
koreader-emulator-i686-w64-mingw32 koreader-emulator-i686-w64-mingw32
koreader-emulator-x86_64-linux-gnu koreader-emulator-x86_64-linux-gnu
koreader-emulator-x86_64-pc-linux-gnu koreader-emulator-x86_64-pc-linux-gnu

@ -1,6 +1,8 @@
language: c language: c
sudo: false # sudo: false
sudo: true
dist: trusty
compiler: compiler:
- gcc - gcc

@ -1 +1 @@
Subproject commit 74e57274ab8d21b99e614bcf4c050e32f407feee Subproject commit 4c41adf0d224b9170fd506c2fcdf639c7b6bc062

@ -1,246 +0,0 @@
--[[--
MD5 hash library.
]]
local ffi = require "ffi"
local bit = require "bit"
local bxor = bit.bxor
local bnot = bit.bnot
local band = bit.band
local bor = bit.bor
local rshift = bit.rshift
local lshift = bit.lshift
local copy = ffi.copy
local fill = ffi.fill
ffi.cdef[[
typedef struct MD5Context {
uint32_t buf[4];
uint32_t bits[2];
unsigned char input[64];
} MD5_CTX;
]]
local function byteReverse(buf, len)
-- TODO: implement for big-endian architectures?
end
local function F1(x, y, z) return bxor(z, band(x, bxor(y, z))) end
local function F2(x, y, z) return F1(z, x, y) end
local function F3(x, y, z) return bxor(x, y, z) end
local function F4(x, y, z) return bxor(y, bor(x, bnot(z))) end
local function MD5STEP(f, w, x, y, z, data, s)
w = w + f(x, y, z) + data
w = bor(lshift(w,s), rshift(w,(32-s)))
w = w + x
return w
end
-- Start MD5 accumulation. Set bit count to 0 and buffer to mysterious
-- initialization constants.
local function MD5Init(ctx)
ctx.buf[0] = 0x67452301
ctx.buf[1] = 0xefcdab89
ctx.buf[2] = 0x98badcfe
ctx.buf[3] = 0x10325476
ctx.bits[0] = 0
ctx.bits[1] = 0
end
local function MD5Transform(buf, input)
local a = buf[0]
local b = buf[1]
local c = buf[2]
local d = buf[3]
a = MD5STEP(F1, a, b, c, d, input[0] + 0xd76aa478, 7)
d = MD5STEP(F1, d, a, b, c, input[1] + 0xe8c7b756, 12)
c = MD5STEP(F1, c, d, a, b, input[2] + 0x242070db, 17)
b = MD5STEP(F1, b, c, d, a, input[3] + 0xc1bdceee, 22)
a = MD5STEP(F1, a, b, c, d, input[4] + 0xf57c0faf, 7)
d = MD5STEP(F1, d, a, b, c, input[5] + 0x4787c62a, 12)
c = MD5STEP(F1, c, d, a, b, input[6] + 0xa8304613, 17)
b = MD5STEP(F1, b, c, d, a, input[7] + 0xfd469501, 22)
a = MD5STEP(F1, a, b, c, d, input[8] + 0x698098d8, 7)
d = MD5STEP(F1, d, a, b, c, input[9] + 0x8b44f7af, 12)
c = MD5STEP(F1, c, d, a, b, input[10] + 0xffff5bb1, 17)
b = MD5STEP(F1, b, c, d, a, input[11] + 0x895cd7be, 22)
a = MD5STEP(F1, a, b, c, d, input[12] + 0x6b901122, 7)
d = MD5STEP(F1, d, a, b, c, input[13] + 0xfd987193, 12)
c = MD5STEP(F1, c, d, a, b, input[14] + 0xa679438e, 17)
b = MD5STEP(F1, b, c, d, a, input[15] + 0x49b40821, 22)
a = MD5STEP(F2, a, b, c, d, input[1] + 0xf61e2562, 5)
d = MD5STEP(F2, d, a, b, c, input[6] + 0xc040b340, 9)
c = MD5STEP(F2, c, d, a, b, input[11] + 0x265e5a51, 14)
b = MD5STEP(F2, b, c, d, a, input[0] + 0xe9b6c7aa, 20)
a = MD5STEP(F2, a, b, c, d, input[5] + 0xd62f105d, 5)
d = MD5STEP(F2, d, a, b, c, input[10] + 0x02441453, 9)
c = MD5STEP(F2, c, d, a, b, input[15] + 0xd8a1e681, 14)
b = MD5STEP(F2, b, c, d, a, input[4] + 0xe7d3fbc8, 20)
a = MD5STEP(F2, a, b, c, d, input[9] + 0x21e1cde6, 5)
d = MD5STEP(F2, d, a, b, c, input[14] + 0xc33707d6, 9)
c = MD5STEP(F2, c, d, a, b, input[3] + 0xf4d50d87, 14)
b = MD5STEP(F2, b, c, d, a, input[8] + 0x455a14ed, 20)
a = MD5STEP(F2, a, b, c, d, input[13] + 0xa9e3e905, 5)
d = MD5STEP(F2, d, a, b, c, input[2] + 0xfcefa3f8, 9)
c = MD5STEP(F2, c, d, a, b, input[7] + 0x676f02d9, 14)
b = MD5STEP(F2, b, c, d, a, input[12] + 0x8d2a4c8a, 20)
a = MD5STEP(F3, a, b, c, d, input[5] + 0xfffa3942, 4)
d = MD5STEP(F3, d, a, b, c, input[8] + 0x8771f681, 11)
c = MD5STEP(F3, c, d, a, b, input[11] + 0x6d9d6122, 16)
b = MD5STEP(F3, b, c, d, a, input[14] + 0xfde5380c, 23)
a = MD5STEP(F3, a, b, c, d, input[1] + 0xa4beea44, 4)
d = MD5STEP(F3, d, a, b, c, input[4] + 0x4bdecfa9, 11)
c = MD5STEP(F3, c, d, a, b, input[7] + 0xf6bb4b60, 16)
b = MD5STEP(F3, b, c, d, a, input[10] + 0xbebfbc70, 23)
a = MD5STEP(F3, a, b, c, d, input[13] + 0x289b7ec6, 4)
d = MD5STEP(F3, d, a, b, c, input[0] + 0xeaa127fa, 11)
c = MD5STEP(F3, c, d, a, b, input[3] + 0xd4ef3085, 16)
b = MD5STEP(F3, b, c, d, a, input[6] + 0x04881d05, 23)
a = MD5STEP(F3, a, b, c, d, input[9] + 0xd9d4d039, 4)
d = MD5STEP(F3, d, a, b, c, input[12] + 0xe6db99e5, 11)
c = MD5STEP(F3, c, d, a, b, input[15] + 0x1fa27cf8, 16)
b = MD5STEP(F3, b, c, d, a, input[2] + 0xc4ac5665, 23)
a = MD5STEP(F4, a, b, c, d, input[0] + 0xf4292244, 6)
d = MD5STEP(F4, d, a, b, c, input[7] + 0x432aff97, 10)
c = MD5STEP(F4, c, d, a, b, input[14] + 0xab9423a7, 15)
b = MD5STEP(F4, b, c, d, a, input[5] + 0xfc93a039, 21)
a = MD5STEP(F4, a, b, c, d, input[12] + 0x655b59c3, 6)
d = MD5STEP(F4, d, a, b, c, input[3] + 0x8f0ccc92, 10)
c = MD5STEP(F4, c, d, a, b, input[10] + 0xffeff47d, 15)
b = MD5STEP(F4, b, c, d, a, input[1] + 0x85845dd1, 21)
a = MD5STEP(F4, a, b, c, d, input[8] + 0x6fa87e4f, 6)
d = MD5STEP(F4, d, a, b, c, input[15] + 0xfe2ce6e0, 10)
c = MD5STEP(F4, c, d, a, b, input[6] + 0xa3014314, 15)
b = MD5STEP(F4, b, c, d, a, input[13] + 0x4e0811a1, 21)
a = MD5STEP(F4, a, b, c, d, input[4] + 0xf7537e82, 6)
d = MD5STEP(F4, d, a, b, c, input[11] + 0xbd3af235, 10)
c = MD5STEP(F4, c, d, a, b, input[2] + 0x2ad7d2bb, 15)
b = MD5STEP(F4, b, c, d, a, input[9] + 0xeb86d391, 21)
buf[0] = band(buf[0] + a, 0xFFFFFFFF)
buf[1] = band(buf[1] + b, 0xFFFFFFFF)
buf[2] = band(buf[2] + c, 0xFFFFFFFF)
buf[3] = band(buf[3] + d, 0xFFFFFFFF)
end
local function MD5Update(ctx, buf, len)
local t
t = ctx.bits[0]
ctx.bits[0] = t + lshift( len, 3)
if (ctx.bits[0] < t) then
ctx.bits[1] = ctx.bits[1] + 1
end
ctx.bits[1] = ctx.bits[1] + rshift(len, 29)
t = band(rshift(t, 3), 0x3f)
if (t > 0) then
local p = ffi.cast("unsigned char *", ctx.input + t)
t = 64 - t
if (len < t) then
copy(p, buf, len)
return
end
copy(p, buf, t)
byteReverse(ctx.input, 16)
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
buf = buf + t
len = len - t
end
while (len >= 64) do
copy(ctx.input, buf, 64)
byteReverse(ctx.input, 16)
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
buf = buf + 64
len = len - 64
end
copy(ctx.input, buf, len)
end
local function MD5Final(digest, ctx)
local count
local p
count = band(rshift(ctx.bits[0], 3), 0x3F)
p = ctx.input + count
p[0] = 0x80
p = p + 1
count = 64 - 1 - count
if (count < 8) then
fill(p, count, 0)
byteReverse(ctx.input, 16)
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
fill(ctx.input, 56, 0)
else
fill(p, count - 8, 0)
end
byteReverse(ctx.input, 14)
ffi.cast("uint32_t *", ctx.input)[14] = ctx.bits[0]
ffi.cast("uint32_t *", ctx.input)[15] = ctx.bits[1]
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
byteReverse(ffi.cast("unsigned char *",ctx.buf), 4)
copy(digest, ctx.buf, 16)
fill(ffi.cast("char *", ctx), ffi.sizeof(ctx), 0)
end
local hex = ffi.new("const char[16]", "0123456789abcdef")
local function bin2str(output, input, len)
if len > 0 then
output[0] = hex[rshift(input[0], 4)]
output[1] = hex[band(input[0], 0xF)]
return bin2str(output+2, input+1, len-1)
end
end
local md5 = {}
--- Create a new md5 hashing instance.
---- @return md5 instance
function md5:new()
self.ctx = ffi.new("MD5_CTX")
MD5Init(self.ctx)
end
--- Feed content to md5 hashing instance.
---- @param luastr Lua string
function md5:update(luastr)
MD5Update(self.ctx, ffi.cast("const char*", luastr), #luastr)
end
--- Calcualte md5 sum.
---- @param luastr Lua string
---- @return md5 sum in Lua string
function md5:sum(luastr)
local buf = ffi.new("char[33]")
local hash = ffi.new("uint8_t[16]")
if luastr then
md5:new()
md5:update(luastr)
end
MD5Final(hash, self.ctx)
bin2str(buf, hash, ffi.sizeof(hash))
return ffi.string(buf)
end
return md5

@ -1,7 +1,7 @@
--[[ --[[
A global LRU cache A global LRU cache
]]-- ]]--
local md5 = require("MD5") local md5 = require("ffi/MD5")
local lfs = require("libs/libkoreader-lfs") local lfs = require("libs/libkoreader-lfs")
local DataStorage = require("datastorage") local DataStorage = require("datastorage")
local DEBUG = require("dbg") local DEBUG = require("dbg")
@ -123,7 +123,7 @@ function Cache:check(key, ItemClass)
end end
return self.cache[key] return self.cache[key]
elseif ItemClass then elseif ItemClass then
local cached = self.cached[md5:sum(key)] local cached = self.cached[md5.sum(key)]
if cached then if cached then
local item = ItemClass:new{} local item = ItemClass:new{}
local ok, msg = pcall(item.load, item, cached) local ok, msg = pcall(item.load, item, cached)
@ -153,14 +153,14 @@ function Cache:serialize()
cached_size = cached_size + (lfs.attributes(file, "size") or 0) cached_size = cached_size + (lfs.attributes(file, "size") or 0)
end end
table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end) table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end)
-- serialize the most recently used cache -- only serialize the most recently used cache
local cache_size = 0 local cache_size = 0
for _, key in ipairs(self.cache_order) do for _, key in ipairs(self.cache_order) do
local cache_item = self.cache[key] local cache_item = self.cache[key]
-- only dump cache item that requests serialization explicitly -- only dump cache item that requests serialization explicitly
if cache_item.persistent and cache_item.dump then if cache_item.persistent and cache_item.dump then
DEBUG("dump cache item", key) DEBUG("dump cache item", key)
cache_size = cache_item:dump(cache_path..md5:sum(key)) or 0 cache_size = cache_item:dump(cache_path..md5.sum(key)) or 0
if cache_size > 0 then break end if cache_size > 0 then break end
end end
end end

@ -109,23 +109,23 @@ end
-- 1048576, 4194304, 16777216, 67108864, 268435456 or 1073741824, appending data -- 1048576, 4194304, 16777216, 67108864, 268435456 or 1073741824, appending data
-- by highlighting in koreader may change the digest value. -- by highlighting in koreader may change the digest value.
function Document:fastDigest() function Document:fastDigest()
local md5 = require("MD5") local md5 = require("ffi/MD5")
local lshift = bit.lshift local lshift = bit.lshift
local file = io.open(self.file, 'rb') local file = io.open(self.file, 'rb')
if file then if file then
local step, size = 1024, 1024 local step, size = 1024, 1024
md5:new() local m = md5.new()
for i = -1, 10 do for i = -1, 10 do
file:seek("set", lshift(step, 2*i)) file:seek("set", lshift(step, 2*i))
local sample = file:read(size) local sample = file:read(size)
if sample then if sample then
md5:update(sample) m:update(sample)
else else
break break
end end
end end
file:close() file:close()
return md5:sum() return m:sum()
end end
end end

10
kodev

@ -42,7 +42,7 @@ OPTIONS:
TARGET: TARGET:
${SUPPORTED_TARGETS}" ${SUPPORTED_TARGETS}"
while [[ $1 == '--'* ]]; do while [[ $1 == '-'* ]]; do
PARAM=`echo $1 | awk -F= '{print $1}'` PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | awk -F= '{print $2}'` VALUE=`echo $1 | awk -F= '{print $2}'`
case $PARAM in case $PARAM in
@ -59,7 +59,7 @@ ${SUPPORTED_TARGETS}"
exit 1 exit 1
;; ;;
esac esac
shift shift 1
done done
case $1 in case $1 in
@ -215,7 +215,7 @@ OPTIONS:
--no-build run reader without rebuilding --no-build run reader without rebuilding
--disable-touch use this if you want to simulate keyboard only devices --disable-touch use this if you want to simulate keyboard only devices
" "
while [[ $1 == '--'* ]]; do while [[ $1 == '-'* ]]; do
PARAM=`echo $1 | awk -F= '{print $1}'` PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | awk -F= '{print $2}'` VALUE=`echo $1 | awk -F= '{print $2}'`
case $PARAM in case $PARAM in
@ -274,7 +274,7 @@ OPTIONS:
--tags=TAGS only run tests with given tags --tags=TAGS only run tests with given tags
" "
while [[ $1 == '--'* ]]; do while [[ $1 == '-'* ]]; do
PARAM=`echo $1 | awk -F= '{print $1}'` PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | awk -F= '{print $2}'` VALUE=`echo $1 | awk -F= '{print $2}'`
case $PARAM in case $PARAM in
@ -308,7 +308,7 @@ OPTIONS:
if [ ! -z $2 ]; then if [ ! -z $2 ]; then
test_path="${test_path}/$2" test_path="${test_path}/$2"
fi fi
busted ${opts} -o verbose_print --exclude-tags=notest ${test_path} busted --lua=./luajit ${opts} -o ./spec/$1/unit/verbose_print --exclude-tags=notest ${test_path}
popd popd
} }

@ -1,7 +1,7 @@
require("commonrequire") require("commonrequire")
local UIManager = require("ui/uimanager") local UIManager = require("ui/uimanager")
local DEBUG = require("dbg") local DEBUG = require("dbg")
local md5 = require("MD5") local md5 = require("ffi/MD5")
--DEBUG:turnOn() --DEBUG:turnOn()
local service = [[ local service = [[
@ -70,7 +70,7 @@ describe("KOSync modules #notest #nocov", function()
req.headers['x-auth-key'] = args.userkey req.headers['x-auth-key'] = args.userkey
end end
-- password should be hashed before submitting to server -- password should be hashed before submitting to server
local username, password = "koreader", md5:sum("koreader") local username, password = "koreader", md5.sum("koreader")
-- fake progress data -- fake progress data
local doc, percentage, progress, device = local doc, percentage, progress, device =
"41cce710f34e5ec21315e19c99821415", -- fast digest of the document "41cce710f34e5ec21315e19c99821415", -- fast digest of the document

@ -1,19 +0,0 @@
require("commonrequire")
local md5 = require("MD5")
describe("MD5 module", function()
it("should calculate correct MD5 hashes", function()
assert.is_equal(md5:sum(""), "d41d8cd98f00b204e9800998ecf8427e")
assert.is_equal(md5:sum("\0"), "93b885adfe0da089cdf634904fd59f71")
assert.is_equal(md5:sum("0123456789abcdefX"), "1b05aba914a8b12315c7ee52b42f3d35")
end)
it("should calculate MD5 sum by updating", function()
md5:new()
md5:update("0123456789")
md5:update("abcdefghij")
local md5sum = md5:sum()
assert.is_equal(md5sum, md5:sum("0123456789abcdefghij"))
end)
end)
Loading…
Cancel
Save