mirror of https://github.com/miguelmota/cointop
dep ensure
Miguel Mota
5 years ago
parent
98b6d2bfbc
commit
8e58d5a9b2
@ -1,24 +0,0 @@
|
||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
*.test
|
||||
*.prof
|
||||
@ -1,10 +0,0 @@
|
||||
go_import_path: go4.org
|
||||
language: go
|
||||
go:
|
||||
- "1.10.x"
|
||||
- tip
|
||||
before_install:
|
||||
- go get -u cloud.google.com/go/storage
|
||||
- cd $HOME/gopath/src/cloud.google.com/go
|
||||
- git reset --hard 4d445121f7d55f37b70187e796b16e64f6eea7a0
|
||||
- cd $HOME/gopath/src/go4.org
|
||||
@ -1,8 +0,0 @@
|
||||
# This is the official list of go4 authors for copyright purposes.
|
||||
# This is distinct from the CONTRIBUTORS file, which is the list of
|
||||
# people who have contributed, even if they don't own the copyright on
|
||||
# their work.
|
||||
|
||||
Mathieu Lonjaret <mathieu.lonjaret@gmail.com>
|
||||
Daniel Theophanes <kardianos@gmail.com>
|
||||
Google
|
||||
@ -1,202 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
@ -1,57 +0,0 @@
|
||||
# go4
|
||||
|
||||
[](https://travis-ci.org/camlistore/go4 "Travis CI")
|
||||
|
||||
[go4.org](http://go4.org) is a collection of packages for
|
||||
Go programmers.
|
||||
|
||||
They started out living in [Perkeep](https://perkeep.org)'s repo
|
||||
and elsewhere but they have nothing to do with Perkeep, so we're
|
||||
moving them here.
|
||||
|
||||
## Details
|
||||
|
||||
* **single repo**. go4 is a single repo. That means things can be
|
||||
changed and rearranged globally atomically with ease and
|
||||
confidence.
|
||||
|
||||
* **no backwards compatibility**. go4 makes no backwards compatibility
|
||||
promises. If you want to use go4, vendor it. And next time you
|
||||
update your vendor tree, update to the latest API if things in go4
|
||||
changed. The plan is to eventually provide tools to make this
|
||||
easier.
|
||||
|
||||
* **forward progress** because we have no backwards compatibility,
|
||||
it's always okay to change things to make things better. That also
|
||||
means the bar for contributions is lower. We don't have to get the
|
||||
API 100% correct in the first commit.
|
||||
|
||||
* **no Go version policy** go4 packages are usually built and tested
|
||||
with the latest Go stable version. However, go4 has no overarching
|
||||
version policy; each package can declare its own set of supported
|
||||
Go versions.
|
||||
|
||||
* **code review** contributions must be code-reviewed. We're trying
|
||||
out Gerrithub, to see if we can find a mix of Github Pull Requests
|
||||
and Gerrit that works well for many people. We'll see.
|
||||
|
||||
* **CLA compliant** contributors must agree to the Google CLA (the
|
||||
same as Go itself). This ensures we can move things into Go as
|
||||
necessary in the future. It also makes lawyers at various
|
||||
companies happy. The CLA is **not** a copyright *assignment*; you
|
||||
retain the copyright on your work. The CLA just says that your
|
||||
work is open source and you have permission to open source it. See
|
||||
https://golang.org/doc/contribute.html#cla
|
||||
|
||||
* **docs, tests, portability** all code should be documented in the
|
||||
normal Go style, have tests, and be portable to different
|
||||
operating systems and architectures. We'll try to get builders in
|
||||
place to help run the tests on different OS/arches. For now we
|
||||
have Travis at least.
|
||||
|
||||
## Contact
|
||||
|
||||
For any question, or communication when a Github issue is not appropriate,
|
||||
please contact the [Perkeep mailing
|
||||
list](https://groups.google.com/forum/#!forum/perkeep).
|
||||
|
||||
@ -1,286 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package bytereplacer provides a utility for replacing parts of byte slices.
|
||||
package bytereplacer // import "go4.org/bytereplacer"
|
||||
|
||||
import "bytes"
|
||||
|
||||
// Replacer replaces a list of strings with replacements.
|
||||
// It is safe for concurrent use by multiple goroutines.
|
||||
type Replacer struct {
|
||||
r replacer
|
||||
}
|
||||
|
||||
// replacer is the interface that a replacement algorithm needs to implement.
|
||||
type replacer interface {
|
||||
// Replace performs all replacements, in-place if possible.
|
||||
Replace(s []byte) []byte
|
||||
}
|
||||
|
||||
// New returns a new Replacer from a list of old, new string pairs.
|
||||
// Replacements are performed in order, without overlapping matches.
|
||||
func New(oldnew ...string) *Replacer {
|
||||
if len(oldnew)%2 == 1 {
|
||||
panic("bytes.NewReplacer: odd argument count")
|
||||
}
|
||||
|
||||
allNewBytes := true
|
||||
for i := 0; i < len(oldnew); i += 2 {
|
||||
if len(oldnew[i]) != 1 {
|
||||
return &Replacer{r: makeGenericReplacer(oldnew)}
|
||||
}
|
||||
if len(oldnew[i+1]) != 1 {
|
||||
allNewBytes = false
|
||||
}
|
||||
}
|
||||
|
||||
if allNewBytes {
|
||||
r := byteReplacer{}
|
||||
for i := range r {
|
||||
r[i] = byte(i)
|
||||
}
|
||||
// The first occurrence of old->new map takes precedence
|
||||
// over the others with the same old string.
|
||||
for i := len(oldnew) - 2; i >= 0; i -= 2 {
|
||||
o := oldnew[i][0]
|
||||
n := oldnew[i+1][0]
|
||||
r[o] = n
|
||||
}
|
||||
return &Replacer{r: &r}
|
||||
}
|
||||
|
||||
return &Replacer{r: makeGenericReplacer(oldnew)}
|
||||
}
|
||||
|
||||
// Replace performs all replacements in-place on s. If the capacity
|
||||
// of s is not sufficient, a new slice is allocated, otherwise Replace
|
||||
// returns s.
|
||||
func (r *Replacer) Replace(s []byte) []byte {
|
||||
return r.r.Replace(s)
|
||||
}
|
||||
|
||||
type trieNode struct {
|
||||
value []byte
|
||||
priority int
|
||||
prefix []byte
|
||||
next *trieNode
|
||||
table []*trieNode
|
||||
}
|
||||
|
||||
func (t *trieNode) add(key, val []byte, priority int, r *genericReplacer) {
|
||||
if len(key) == 0 {
|
||||
if t.priority == 0 {
|
||||
t.value = val
|
||||
t.priority = priority
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(t.prefix) > 0 {
|
||||
// Need to split the prefix among multiple nodes.
|
||||
var n int // length of the longest common prefix
|
||||
for ; n < len(t.prefix) && n < len(key); n++ {
|
||||
if t.prefix[n] != key[n] {
|
||||
break
|
||||
}
|
||||
}
|
||||
if n == len(t.prefix) {
|
||||
t.next.add(key[n:], val, priority, r)
|
||||
} else if n == 0 {
|
||||
// First byte differs, start a new lookup table here. Looking up
|
||||
// what is currently t.prefix[0] will lead to prefixNode, and
|
||||
// looking up key[0] will lead to keyNode.
|
||||
var prefixNode *trieNode
|
||||
if len(t.prefix) == 1 {
|
||||
prefixNode = t.next
|
||||
} else {
|
||||
prefixNode = &trieNode{
|
||||
prefix: t.prefix[1:],
|
||||
next: t.next,
|
||||
}
|
||||
}
|
||||
keyNode := new(trieNode)
|
||||
t.table = make([]*trieNode, r.tableSize)
|
||||
t.table[r.mapping[t.prefix[0]]] = prefixNode
|
||||
t.table[r.mapping[key[0]]] = keyNode
|
||||
t.prefix = nil
|
||||
t.next = nil
|
||||
keyNode.add(key[1:], val, priority, r)
|
||||
} else {
|
||||
// Insert new node after the common section of the prefix.
|
||||
next := &trieNode{
|
||||
prefix: t.prefix[n:],
|
||||
next: t.next,
|
||||
}
|
||||
t.prefix = t.prefix[:n]
|
||||
t.next = next
|
||||
next.add(key[n:], val, priority, r)
|
||||
}
|
||||
} else if t.table != nil {
|
||||
// Insert into existing table.
|
||||
m := r.mapping[key[0]]
|
||||
if t.table[m] == nil {
|
||||
t.table[m] = new(trieNode)
|
||||
}
|
||||
t.table[m].add(key[1:], val, priority, r)
|
||||
} else {
|
||||
t.prefix = key
|
||||
t.next = new(trieNode)
|
||||
t.next.add(nil, val, priority, r)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *genericReplacer) lookup(s []byte, ignoreRoot bool) (val []byte, keylen int, found bool) {
|
||||
// Iterate down the trie to the end, and grab the value and keylen with
|
||||
// the highest priority.
|
||||
bestPriority := 0
|
||||
node := &r.root
|
||||
n := 0
|
||||
for node != nil {
|
||||
if node.priority > bestPriority && !(ignoreRoot && node == &r.root) {
|
||||
bestPriority = node.priority
|
||||
val = node.value
|
||||
keylen = n
|
||||
found = true
|
||||
}
|
||||
|
||||
if len(s) == 0 {
|
||||
break
|
||||
}
|
||||
if node.table != nil {
|
||||
index := r.mapping[s[0]]
|
||||
if int(index) == r.tableSize {
|
||||
break
|
||||
}
|
||||
node = node.table[index]
|
||||
s = s[1:]
|
||||
n++
|
||||
} else if len(node.prefix) > 0 && bytes.HasPrefix(s, node.prefix) {
|
||||
n += len(node.prefix)
|
||||
s = s[len(node.prefix):]
|
||||
node = node.next
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// genericReplacer is the fully generic algorithm.
|
||||
// It's used as a fallback when nothing faster can be used.
|
||||
type genericReplacer struct {
|
||||
root trieNode
|
||||
// tableSize is the size of a trie node's lookup table. It is the number
|
||||
// of unique key bytes.
|
||||
tableSize int
|
||||
// mapping maps from key bytes to a dense index for trieNode.table.
|
||||
mapping [256]byte
|
||||
}
|
||||
|
||||
func makeGenericReplacer(oldnew []string) *genericReplacer {
|
||||
r := new(genericReplacer)
|
||||
// Find each byte used, then assign them each an index.
|
||||
for i := 0; i < len(oldnew); i += 2 {
|
||||
key := oldnew[i]
|
||||
for j := 0; j < len(key); j++ {
|
||||
r.mapping[key[j]] = 1
|
||||
}
|
||||
}
|
||||
|
||||
for _, b := range r.mapping {
|
||||
r.tableSize += int(b)
|
||||
}
|
||||
|
||||
var index byte
|
||||
for i, b := range r.mapping {
|
||||
if b == 0 {
|
||||
r.mapping[i] = byte(r.tableSize)
|
||||
} else {
|
||||
r.mapping[i] = index
|
||||
index++
|
||||
}
|
||||
}
|
||||
// Ensure root node uses a lookup table (for performance).
|
||||
r.root.table = make([]*trieNode, r.tableSize)
|
||||
|
||||
for i := 0; i < len(oldnew); i += 2 {
|
||||
r.root.add([]byte(oldnew[i]), []byte(oldnew[i+1]), len(oldnew)-i, r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *genericReplacer) Replace(s []byte) []byte {
|
||||
var last int
|
||||
var prevMatchEmpty bool
|
||||
dst := s[:0]
|
||||
grown := false
|
||||
for i := 0; i <= len(s); {
|
||||
// Fast path: s[i] is not a prefix of any pattern.
|
||||
if i != len(s) && r.root.priority == 0 {
|
||||
index := int(r.mapping[s[i]])
|
||||
if index == r.tableSize || r.root.table[index] == nil {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Ignore the empty match iff the previous loop found the empty match.
|
||||
val, keylen, match := r.lookup(s[i:], prevMatchEmpty)
|
||||
prevMatchEmpty = match && keylen == 0
|
||||
if match {
|
||||
dst = append(dst, s[last:i]...)
|
||||
if diff := len(val) - keylen; grown || diff < 0 {
|
||||
dst = append(dst, val...)
|
||||
i += keylen
|
||||
} else if diff <= cap(s)-len(s) {
|
||||
// The replacement is larger than the original, but can still fit in the original buffer.
|
||||
copy(s[i+len(val):cap(dst)], s[i+keylen:])
|
||||
dst = append(dst, val...)
|
||||
s = s[:len(s)+diff]
|
||||
i += len(val)
|
||||
} else {
|
||||
// The output will grow larger than the original buffer. Allocate a new one.
|
||||
grown = true
|
||||
newDst := make([]byte, len(dst), cap(dst)+diff)
|
||||
copy(newDst, dst)
|
||||
dst = newDst
|
||||
|
||||
dst = append(dst, val...)
|
||||
i += keylen
|
||||
}
|
||||
last = i
|
||||
continue
|
||||
}
|
||||
i++
|
||||
}
|
||||
if last != len(s) {
|
||||
dst = append(dst, s[last:]...)
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// byteReplacer is the implementation that's used when all the "old"
|
||||
// and "new" values are single ASCII bytes.
|
||||
// The array contains replacement bytes indexed by old byte.
|
||||
type byteReplacer [256]byte
|
||||
|
||||
func (r *byteReplacer) Replace(s []byte) []byte {
|
||||
for i, b := range s {
|
||||
s[i] = r[b]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@ -1,423 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package bytereplacer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var htmlEscaper = New(
|
||||
"&", "&",
|
||||
"<", "<",
|
||||
">", ">",
|
||||
`"`, """,
|
||||
"'", "'",
|
||||
)
|
||||
|
||||
var htmlUnescaper = New(
|
||||
"&", "&",
|
||||
"<", "<",
|
||||
">", ">",
|
||||
""", `"`,
|
||||
"'", "'",
|
||||
)
|
||||
|
||||
var capitalLetters = New("a", "A", "b", "B")
|
||||
|
||||
func TestReplacer(t *testing.T) {
|
||||
type testCase struct {
|
||||
r *Replacer
|
||||
in, out string
|
||||
}
|
||||
var testCases []testCase
|
||||
|
||||
// str converts 0xff to "\xff". This isn't just string(b) since that converts to UTF-8.
|
||||
str := func(b byte) string {
|
||||
return string([]byte{b})
|
||||
}
|
||||
var s []string
|
||||
|
||||
// inc maps "\x00"->"\x01", ..., "a"->"b", "b"->"c", ..., "\xff"->"\x00".
|
||||
s = nil
|
||||
for i := 0; i < 256; i++ {
|
||||
s = append(s, str(byte(i)), str(byte(i+1)))
|
||||
}
|
||||
inc := New(s...)
|
||||
|
||||
// Test cases with 1-byte old strings, 1-byte new strings.
|
||||
testCases = append(testCases,
|
||||
testCase{capitalLetters, "brad", "BrAd"},
|
||||
testCase{capitalLetters, strings.Repeat("a", (32<<10)+123), strings.Repeat("A", (32<<10)+123)},
|
||||
testCase{capitalLetters, "", ""},
|
||||
|
||||
testCase{inc, "brad", "csbe"},
|
||||
testCase{inc, "\x00\xff", "\x01\x00"},
|
||||
testCase{inc, "", ""},
|
||||
|
||||
testCase{New("a", "1", "a", "2"), "brad", "br1d"},
|
||||
)
|
||||
|
||||
// repeat maps "a"->"a", "b"->"bb", "c"->"ccc", ...
|
||||
s = nil
|
||||
for i := 0; i < 256; i++ {
|
||||
n := i + 1 - 'a'
|
||||
if n < 1 {
|
||||
n = 1
|
||||
}
|
||||
s = append(s, str(byte(i)), strings.Repeat(str(byte(i)), n))
|
||||
}
|
||||
repeat := New(s...)
|
||||
|
||||
// Test cases with 1-byte old strings, variable length new strings.
|
||||
testCases = append(testCases,
|
||||
testCase{htmlEscaper, "No changes", "No changes"},
|
||||
testCase{htmlEscaper, "I <3 escaping & stuff", "I <3 escaping & stuff"},
|
||||
testCase{htmlEscaper, "&&&", "&&&"},
|
||||
testCase{htmlEscaper, "", ""},
|
||||
|
||||
testCase{repeat, "brad", "bbrrrrrrrrrrrrrrrrrradddd"},
|
||||
testCase{repeat, "abba", "abbbba"},
|
||||
testCase{repeat, "", ""},
|
||||
|
||||
testCase{New("a", "11", "a", "22"), "brad", "br11d"},
|
||||
)
|
||||
|
||||
// The remaining test cases have variable length old strings.
|
||||
|
||||
testCases = append(testCases,
|
||||
testCase{htmlUnescaper, "&amp;", "&"},
|
||||
testCase{htmlUnescaper, "<b>HTML's neat</b>", "<b>HTML's neat</b>"},
|
||||
testCase{htmlUnescaper, "", ""},
|
||||
|
||||
testCase{New("a", "1", "a", "2", "xxx", "xxx"), "brad", "br1d"},
|
||||
|
||||
testCase{New("a", "1", "aa", "2", "aaa", "3"), "aaaa", "1111"},
|
||||
|
||||
testCase{New("aaa", "3", "aa", "2", "a", "1"), "aaaa", "31"},
|
||||
)
|
||||
|
||||
// gen1 has multiple old strings of variable length. There is no
|
||||
// overall non-empty common prefix, but some pairwise common prefixes.
|
||||
gen1 := New(
|
||||
"aaa", "3[aaa]",
|
||||
"aa", "2[aa]",
|
||||
"a", "1[a]",
|
||||
"i", "i",
|
||||
"longerst", "most long",
|
||||
"longer", "medium",
|
||||
"long", "short",
|
||||
"xx", "xx",
|
||||
"x", "X",
|
||||
"X", "Y",
|
||||
"Y", "Z",
|
||||
)
|
||||
testCases = append(testCases,
|
||||
testCase{gen1, "fooaaabar", "foo3[aaa]b1[a]r"},
|
||||
testCase{gen1, "long, longerst, longer", "short, most long, medium"},
|
||||
testCase{gen1, "xxxxx", "xxxxX"},
|
||||
testCase{gen1, "XiX", "YiY"},
|
||||
testCase{gen1, "", ""},
|
||||
)
|
||||
|
||||
// gen2 has multiple old strings with no pairwise common prefix.
|
||||
gen2 := New(
|
||||
"roses", "red",
|
||||
"violets", "blue",
|
||||
"sugar", "sweet",
|
||||
)
|
||||
testCases = append(testCases,
|
||||
testCase{gen2, "roses are red, violets are blue...", "red are red, blue are blue..."},
|
||||
testCase{gen2, "", ""},
|
||||
)
|
||||
|
||||
// gen3 has multiple old strings with an overall common prefix.
|
||||
gen3 := New(
|
||||
"abracadabra", "poof",
|
||||
"abracadabrakazam", "splat",
|
||||
"abraham", "lincoln",
|
||||
"abrasion", "scrape",
|
||||
"abraham", "isaac",
|
||||
)
|
||||
testCases = append(testCases,
|
||||
testCase{gen3, "abracadabrakazam abraham", "poofkazam lincoln"},
|
||||
testCase{gen3, "abrasion abracad", "scrape abracad"},
|
||||
testCase{gen3, "abba abram abrasive", "abba abram abrasive"},
|
||||
testCase{gen3, "", ""},
|
||||
)
|
||||
|
||||
// foo{1,2,3,4} have multiple old strings with an overall common prefix
|
||||
// and 1- or 2- byte extensions from the common prefix.
|
||||
foo1 := New(
|
||||
"foo1", "A",
|
||||
"foo2", "B",
|
||||
"foo3", "C",
|
||||
)
|
||||
foo2 := New(
|
||||
"foo1", "A",
|
||||
"foo2", "B",
|
||||
"foo31", "C",
|
||||
"foo32", "D",
|
||||
)
|
||||
foo3 := New(
|
||||
"foo11", "A",
|
||||
"foo12", "B",
|
||||
"foo31", "C",
|
||||
"foo32", "D",
|
||||
)
|
||||
foo4 := New(
|
||||
"foo12", "B",
|
||||
"foo32", "D",
|
||||
)
|
||||
testCases = append(testCases,
|
||||
testCase{foo1, "fofoofoo12foo32oo", "fofooA2C2oo"},
|
||||
testCase{foo1, "", ""},
|
||||
|
||||
testCase{foo2, "fofoofoo12foo32oo", "fofooA2Doo"},
|
||||
testCase{foo2, "", ""},
|
||||
|
||||
testCase{foo3, "fofoofoo12foo32oo", "fofooBDoo"},
|
||||
testCase{foo3, "", ""},
|
||||
|
||||
testCase{foo4, "fofoofoo12foo32oo", "fofooBDoo"},
|
||||
testCase{foo4, "", ""},
|
||||
)
|
||||
|
||||
// genAll maps "\x00\x01\x02...\xfe\xff" to "[all]", amongst other things.
|
||||
allBytes := make([]byte, 256)
|
||||
for i := range allBytes {
|
||||
allBytes[i] = byte(i)
|
||||
}
|
||||
allString := string(allBytes)
|
||||
genAll := New(
|
||||
allString, "[all]",
|
||||
"\xff", "[ff]",
|
||||
"\x00", "[00]",
|
||||
)
|
||||
testCases = append(testCases,
|
||||
testCase{genAll, allString, "[all]"},
|
||||
testCase{genAll, "a\xff" + allString + "\x00", "a[ff][all][00]"},
|
||||
testCase{genAll, "", ""},
|
||||
)
|
||||
|
||||
// Test cases with empty old strings.
|
||||
|
||||
blankToX1 := New("", "X")
|
||||
blankToX2 := New("", "X", "", "")
|
||||
blankHighPriority := New("", "X", "o", "O")
|
||||
blankLowPriority := New("o", "O", "", "X")
|
||||
blankNoOp1 := New("", "")
|
||||
blankNoOp2 := New("", "", "", "A")
|
||||
blankFoo := New("", "X", "foobar", "R", "foobaz", "Z")
|
||||
testCases = append(testCases,
|
||||
testCase{blankToX1, "foo", "XfXoXoX"},
|
||||
testCase{blankToX1, "", "X"},
|
||||
|
||||
testCase{blankToX2, "foo", "XfXoXoX"},
|
||||
testCase{blankToX2, "", "X"},
|
||||
|
||||
testCase{blankHighPriority, "oo", "XOXOX"},
|
||||
testCase{blankHighPriority, "ii", "XiXiX"},
|
||||
testCase{blankHighPriority, "oiio", "XOXiXiXOX"},
|
||||
testCase{blankHighPriority, "iooi", "XiXOXOXiX"},
|
||||
testCase{blankHighPriority, "", "X"},
|
||||
|
||||
testCase{blankLowPriority, "oo", "OOX"},
|
||||
testCase{blankLowPriority, "ii", "XiXiX"},
|
||||
testCase{blankLowPriority, "oiio", "OXiXiOX"},
|
||||
testCase{blankLowPriority, "iooi", "XiOOXiX"},
|
||||
testCase{blankLowPriority, "", "X"},
|
||||
|
||||
testCase{blankNoOp1, "foo", "foo"},
|
||||
testCase{blankNoOp1, "", ""},
|
||||
|
||||
testCase{blankNoOp2, "foo", "foo"},
|
||||
testCase{blankNoOp2, "", ""},
|
||||
|
||||
testCase{blankFoo, "foobarfoobaz", "XRXZX"},
|
||||
testCase{blankFoo, "foobar-foobaz", "XRX-XZX"},
|
||||
testCase{blankFoo, "", "X"},
|
||||
)
|
||||
|
||||
// single string replacer
|
||||
|
||||
abcMatcher := New("abc", "[match]")
|
||||
|
||||
testCases = append(testCases,
|
||||
testCase{abcMatcher, "", ""},
|
||||
testCase{abcMatcher, "ab", "ab"},
|
||||
testCase{abcMatcher, "abc", "[match]"},
|
||||
testCase{abcMatcher, "abcd", "[match]d"},
|
||||
testCase{abcMatcher, "cabcabcdabca", "c[match][match]d[match]a"},
|
||||
)
|
||||
|
||||
// Issue 6659 cases (more single string replacer)
|
||||
|
||||
noHello := New("Hello", "")
|
||||
testCases = append(testCases,
|
||||
testCase{noHello, "Hello", ""},
|
||||
testCase{noHello, "Hellox", "x"},
|
||||
testCase{noHello, "xHello", "x"},
|
||||
testCase{noHello, "xHellox", "xx"},
|
||||
)
|
||||
|
||||
// No-arg test cases.
|
||||
|
||||
nop := New()
|
||||
testCases = append(testCases,
|
||||
testCase{nop, "abc", "abc"},
|
||||
testCase{nop, "", ""},
|
||||
)
|
||||
|
||||
// Run the test cases.
|
||||
|
||||
for i, tc := range testCases {
|
||||
{
|
||||
// Replace with len(in) == cap(in)
|
||||
in := make([]byte, len(tc.in))
|
||||
copy(in, tc.in)
|
||||
if s := string(tc.r.Replace(in)); s != tc.out {
|
||||
t.Errorf("%d. Replace(%q /* len == cap */) = %q, want %q", i, tc.in, s, tc.out)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// Replace with len(in) < cap(in)
|
||||
in := make([]byte, len(tc.in), len(tc.in)*2)
|
||||
copy(in, tc.in)
|
||||
if s := string(tc.r.Replace(in)); s != tc.out {
|
||||
t.Errorf("%d. Replace(%q /* len < cap */) = %q, want %q", i, tc.in, s, tc.out)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGenericNoMatch(b *testing.B) {
|
||||
str := []byte(strings.Repeat("A", 100) + strings.Repeat("B", 100))
|
||||
generic := New("a", "A", "b", "B", "12", "123") // varying lengths forces generic
|
||||
for i := 0; i < b.N; i++ {
|
||||
generic.Replace(str)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGenericMatch1(b *testing.B) {
|
||||
str := []byte(strings.Repeat("a", 100) + strings.Repeat("b", 100))
|
||||
generic := New("a", "A", "b", "B", "12", "123")
|
||||
for i := 0; i < b.N; i++ {
|
||||
generic.Replace(str)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkGenericMatch2(b *testing.B) {
|
||||
str := bytes.Repeat([]byte("It's <b>HTML</b>!"), 100)
|
||||
for i := 0; i < b.N; i++ {
|
||||
htmlUnescaper.Replace(str)
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkSingleString(b *testing.B, pattern, text string) {
|
||||
r := New(pattern, "[match]")
|
||||
buf := make([]byte, len(text), len(text)*7)
|
||||
b.SetBytes(int64(len(text)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(buf, text)
|
||||
r.Replace(buf)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSingleMaxSkipping(b *testing.B) {
|
||||
benchmarkSingleString(b, strings.Repeat("b", 25), strings.Repeat("a", 10000))
|
||||
}
|
||||
|
||||
func BenchmarkSingleLongSuffixFail(b *testing.B) {
|
||||
benchmarkSingleString(b, "b"+strings.Repeat("a", 500), strings.Repeat("a", 1002))
|
||||
}
|
||||
|
||||
func BenchmarkSingleMatch(b *testing.B) {
|
||||
benchmarkSingleString(b, "abcdef", strings.Repeat("abcdefghijklmno", 1000))
|
||||
}
|
||||
|
||||
func benchmarkReplacer(b *testing.B, r *Replacer, str string) {
|
||||
buf := make([]byte, len(str))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(buf, str)
|
||||
r.Replace(buf)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkByteByteNoMatch(b *testing.B) {
|
||||
benchmarkReplacer(b, capitalLetters, strings.Repeat("A", 100)+strings.Repeat("B", 100))
|
||||
}
|
||||
|
||||
func BenchmarkByteByteMatch(b *testing.B) {
|
||||
benchmarkReplacer(b, capitalLetters, strings.Repeat("a", 100)+strings.Repeat("b", 100))
|
||||
}
|
||||
|
||||
func BenchmarkByteStringMatch(b *testing.B) {
|
||||
benchmarkReplacer(b, htmlEscaper, "<"+strings.Repeat("a", 99)+strings.Repeat("b", 99)+">")
|
||||
}
|
||||
|
||||
func BenchmarkHTMLEscapeNew(b *testing.B) {
|
||||
benchmarkReplacer(b, htmlEscaper, "I <3 to escape HTML & other text too.")
|
||||
}
|
||||
|
||||
func BenchmarkHTMLEscapeOld(b *testing.B) {
|
||||
str := "I <3 to escape HTML & other text too."
|
||||
buf := make([]byte, len(str))
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(buf, str)
|
||||
oldHTMLEscape(buf)
|
||||
}
|
||||
}
|
||||
|
||||
// The http package's old HTML escaping function in bytes form.
|
||||
func oldHTMLEscape(s []byte) []byte {
|
||||
s = bytes.Replace(s, []byte("&"), []byte("&"), -1)
|
||||
s = bytes.Replace(s, []byte("<"), []byte("<"), -1)
|
||||
s = bytes.Replace(s, []byte(">"), []byte(">"), -1)
|
||||
s = bytes.Replace(s, []byte(`"`), []byte("""), -1)
|
||||
s = bytes.Replace(s, []byte("'"), []byte("'"), -1)
|
||||
return s
|
||||
}
|
||||
|
||||
// BenchmarkByteByteReplaces compares byteByteImpl against multiple Replaces.
|
||||
func BenchmarkByteByteReplaces(b *testing.B) {
|
||||
str := strings.Repeat("a", 100) + strings.Repeat("b", 100)
|
||||
for i := 0; i < b.N; i++ {
|
||||
bytes.Replace(bytes.Replace([]byte(str), []byte{'a'}, []byte{'A'}, -1), []byte{'b'}, []byte{'B'}, -1)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkByteByteMap compares byteByteImpl against Map.
|
||||
func BenchmarkByteByteMap(b *testing.B) {
|
||||
str := strings.Repeat("a", 100) + strings.Repeat("b", 100)
|
||||
fn := func(r rune) rune {
|
||||
switch r {
|
||||
case 'a':
|
||||
return 'A'
|
||||
case 'b':
|
||||
return 'B'
|
||||
}
|
||||
return r
|
||||
}
|
||||
for i := 0; i < b.N; i++ {
|
||||
bytes.Map(fn, []byte(str))
|
||||
}
|
||||
}
|
||||
@ -1,457 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package cloudlaunch helps binaries run themselves on The Cloud, copying
|
||||
// themselves to GCE.
|
||||
package cloudlaunch // import "go4.org/cloud/cloudlaunch"
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go4.org/cloud/google/gceutil"
|
||||
|
||||
"cloud.google.com/go/compute/metadata"
|
||||
"cloud.google.com/go/storage"
|
||||
"golang.org/x/net/context"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
compute "google.golang.org/api/compute/v1"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
storageapi "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
func readFile(v string) string {
|
||||
slurp, err := ioutil.ReadFile(v)
|
||||
if err != nil {
|
||||
log.Fatalf("Error reading %s: %v", v, err)
|
||||
}
|
||||
return strings.TrimSpace(string(slurp))
|
||||
}
|
||||
|
||||
const baseConfig = `#cloud-config
|
||||
coreos:
|
||||
update:
|
||||
group: stable
|
||||
reboot-strategy: $REBOOT
|
||||
units:
|
||||
- name: $NAME.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=$NAME service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStartPre=/bin/sh -c 'mkdir -p /opt/bin && /usr/bin/curl --silent -f -o /opt/bin/$NAME $URL?$(date +%s) && chmod +x /opt/bin/$NAME'
|
||||
ExecStart=/opt/bin/$NAME
|
||||
RestartSec=10
|
||||
Restart=always
|
||||
StartLimitInterval=0
|
||||
|
||||
[Install]
|
||||
WantedBy=network-online.target
|
||||
`
|
||||
|
||||
// RestartPolicy controls whether the binary automatically restarts.
|
||||
type RestartPolicy int
|
||||
|
||||
const (
|
||||
RestartOnUpdates RestartPolicy = iota
|
||||
RestartNever
|
||||
// TODO: more graceful restarts; make systemd own listening on network sockets,
|
||||
// don't break connections.
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
// Name is the name of a service to run.
|
||||
// This is the name of the systemd service (without .service)
|
||||
// and the name of the GCE instance.
|
||||
Name string
|
||||
|
||||
// RestartPolicy controls whether the binary automatically restarts
|
||||
// on updates. The zero value means automatic.
|
||||
RestartPolicy RestartPolicy
|
||||
|
||||
// UpdateStrategy sets the CoreOS automatic update strategy, and the
|
||||
// associated reboots. Possible values are "best-effort", "etcd-lock",
|
||||
// "reboot", "off", with "best-effort" being the default. See
|
||||
// https://coreos.com/os/docs/latest/update-strategies.html
|
||||
UpdateStrategy string
|
||||
|
||||
// BinaryBucket and BinaryObject are the GCS bucket and object
|
||||
// within that bucket containing the Linux binary to download
|
||||
// on boot and occasionally run. This binary must be public
|
||||
// (at least for now).
|
||||
BinaryBucket string
|
||||
BinaryObject string // defaults to Name
|
||||
|
||||
GCEProjectID string
|
||||
Zone string // defaults to us-central1-f
|
||||
SSD bool
|
||||
|
||||
Scopes []string // any additional scopes
|
||||
|
||||
MachineType string
|
||||
InstanceName string
|
||||
}
|
||||
|
||||
// cloudLaunch is a launch of a Config.
|
||||
type cloudLaunch struct {
|
||||
*Config
|
||||
oauthClient *http.Client
|
||||
computeService *compute.Service
|
||||
}
|
||||
|
||||
func (c *Config) binaryURL() string {
|
||||
return "https://storage.googleapis.com/" + c.BinaryBucket + "/" + c.binaryObject()
|
||||
}
|
||||
|
||||
func (c *Config) instName() string { return c.Name } // for now
|
||||
func (c *Config) zone() string { return strDefault(c.Zone, "us-central1-f") }
|
||||
func (c *Config) machineType() string { return strDefault(c.MachineType, "g1-small") }
|
||||
func (c *Config) binaryObject() string { return strDefault(c.BinaryObject, c.Name) }
|
||||
func (c *Config) updateStrategy() string { return strDefault(c.UpdateStrategy, "best-effort") }
|
||||
|
||||
func (c *Config) projectAPIURL() string {
|
||||
return "https://www.googleapis.com/compute/v1/projects/" + c.GCEProjectID
|
||||
}
|
||||
func (c *Config) machineTypeURL() string {
|
||||
return c.projectAPIURL() + "/zones/" + c.zone() + "/machineTypes/" + c.machineType()
|
||||
}
|
||||
|
||||
func strDefault(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var (
|
||||
doLaunch = flag.Bool("cloudlaunch", false, "Deploy or update this binary to the cloud. Must be on Linux, for now.")
|
||||
)
|
||||
|
||||
func (c *Config) MaybeDeploy() {
|
||||
flag.Parse()
|
||||
if !*doLaunch {
|
||||
go c.restartLoop()
|
||||
return
|
||||
}
|
||||
defer os.Exit(1) // backup, in case we return without Fatal or os.Exit later
|
||||
|
||||
if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
|
||||
log.Fatal("Can only use --cloudlaunch on linux/amd64, for now.")
|
||||
}
|
||||
|
||||
if c.GCEProjectID == "" {
|
||||
log.Fatal("cloudconfig.GCEProjectID is empty")
|
||||
}
|
||||
filename := filepath.Join(os.Getenv("HOME"), "keys", c.GCEProjectID+".key.json")
|
||||
log.Printf("Using OAuth config from JSON service file: %s", filename)
|
||||
jwtConf, err := google.JWTConfigFromJSON([]byte(readFile(filename)), append([]string{
|
||||
storageapi.DevstorageFullControlScope,
|
||||
compute.ComputeScope,
|
||||
"https://www.googleapis.com/auth/cloud-platform",
|
||||
}, c.Scopes...)...)
|
||||
if err != nil {
|
||||
log.Fatalf("ConfigFromJSON: %v", err)
|
||||
}
|
||||
|
||||
cl := &cloudLaunch{
|
||||
Config: c,
|
||||
oauthClient: jwtConf.Client(oauth2.NoContext),
|
||||
}
|
||||
cl.computeService, _ = compute.New(cl.oauthClient)
|
||||
|
||||
cl.uploadBinary()
|
||||
cl.createInstance()
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func (c *Config) restartLoop() {
|
||||
if !metadata.OnGCE() {
|
||||
return
|
||||
}
|
||||
if c.RestartPolicy == RestartNever {
|
||||
return
|
||||
}
|
||||
url := c.binaryURL()
|
||||
var lastEtag string
|
||||
for {
|
||||
res, err := http.Head(url + "?" + fmt.Sprint(time.Now().Unix()))
|
||||
if err != nil {
|
||||
log.Printf("Warning: %v", err)
|
||||
time.Sleep(15 * time.Second)
|
||||
continue
|
||||
}
|
||||
etag := res.Header.Get("Etag")
|
||||
if etag == "" {
|
||||
log.Printf("Warning, no ETag in response: %v", res)
|
||||
time.Sleep(15 * time.Second)
|
||||
continue
|
||||
}
|
||||
if lastEtag != "" && etag != lastEtag {
|
||||
log.Printf("Binary updated; restarting.")
|
||||
// TODO: more graceful restart, letting systemd own the network connections.
|
||||
// Then we can finish up requests here.
|
||||
os.Exit(0)
|
||||
}
|
||||
lastEtag = etag
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadBinary uploads the currently-running Linux binary.
|
||||
// It crashes if it fails.
|
||||
func (cl *cloudLaunch) uploadBinary() {
|
||||
ctx := context.Background()
|
||||
if cl.BinaryBucket == "" {
|
||||
log.Fatal("cloudlaunch: Config.BinaryBucket is empty")
|
||||
}
|
||||
stoClient, err := storage.NewClient(ctx, option.WithHTTPClient(cl.oauthClient))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
w := stoClient.Bucket(cl.BinaryBucket).Object(cl.binaryObject()).NewWriter(ctx)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
w.ACL = []storage.ACLRule{
|
||||
// If you don't give the owners access, the web UI seems to
|
||||
// have a bug and doesn't have access to see that it's public, so
|
||||
// won't render the "Shared Publicly" link. So we do that, even
|
||||
// though it's dumb and unnecessary otherwise:
|
||||
{
|
||||
Entity: storage.ACLEntity("project-owners-" + cl.GCEProjectID),
|
||||
Role: storage.RoleOwner,
|
||||
},
|
||||
// Public, so our systemd unit can get it easily:
|
||||
{
|
||||
Entity: storage.AllUsers,
|
||||
Role: storage.RoleReader,
|
||||
},
|
||||
}
|
||||
w.CacheControl = "no-cache"
|
||||
selfPath := getSelfPath()
|
||||
log.Printf("Uploading %q to %v", selfPath, cl.binaryURL())
|
||||
f, err := os.Open(selfPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
n, err := io.Copy(w, f)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("Uploaded %d bytes", n)
|
||||
}
|
||||
|
||||
func getSelfPath() string {
|
||||
if runtime.GOOS != "linux" {
|
||||
panic("TODO")
|
||||
}
|
||||
v, err := os.Readlink("/proc/self/exe")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func zoneInRegion(zone, regionURL string) bool {
|
||||
if zone == "" {
|
||||
panic("empty zone")
|
||||
}
|
||||
if regionURL == "" {
|
||||
panic("empty regionURL")
|
||||
}
|
||||
// zone is like "us-central1-f"
|
||||
// regionURL is like "https://www.googleapis.com/compute/v1/projects/camlistore-website/regions/us-central1"
|
||||
region := path.Base(regionURL) // "us-central1"
|
||||
if region == "" {
|
||||
panic("empty region")
|
||||
}
|
||||
return strings.HasPrefix(zone, region)
|
||||
}
|
||||
|
||||
// findIP finds an IP address to use, or returns the empty string if none is found.
|
||||
// It tries to find a reserved one in the same region where the name of the reserved IP
|
||||
// is "NAME-ip" and the IP is not in use.
|
||||
func (cl *cloudLaunch) findIP() string {
|
||||
// Try to find it by name.
|
||||
aggAddrList, err := cl.computeService.Addresses.AggregatedList(cl.GCEProjectID).Do()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
|
||||
var ip string
|
||||
IPLoop:
|
||||
for _, asl := range aggAddrList.Items {
|
||||
for _, addr := range asl.Addresses {
|
||||
log.Printf(" addr: %#v", addr)
|
||||
if addr.Name == cl.Name+"-ip" && addr.Status == "RESERVED" && zoneInRegion(cl.zone(), addr.Region) {
|
||||
ip = addr.Address
|
||||
break IPLoop
|
||||
}
|
||||
}
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
func (cl *cloudLaunch) createInstance() {
|
||||
inst := cl.lookupInstance()
|
||||
if inst != nil {
|
||||
log.Printf("Instance exists; not re-creating.")
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("Instance doesn't exist; creating...")
|
||||
|
||||
ip := cl.findIP()
|
||||
log.Printf("Found IP: %v", ip)
|
||||
|
||||
cloudConfig := strings.NewReplacer(
|
||||
"$NAME", cl.Name,
|
||||
"$URL", cl.binaryURL(),
|
||||
"$REBOOT", cl.updateStrategy(),
|
||||
).Replace(baseConfig)
|
||||
|
||||
instance := &compute.Instance{
|
||||
Name: cl.instName(),
|
||||
Description: cl.Name,
|
||||
MachineType: cl.machineTypeURL(),
|
||||
Disks: []*compute.AttachedDisk{cl.instanceDisk()},
|
||||
Tags: &compute.Tags{
|
||||
Items: []string{"http-server", "https-server"},
|
||||
},
|
||||
Metadata: &compute.Metadata{
|
||||
Items: []*compute.MetadataItems{
|
||||
{
|
||||
Key: "user-data",
|
||||
Value: googleapi.String(cloudConfig),
|
||||
},
|
||||
},
|
||||
},
|
||||
NetworkInterfaces: []*compute.NetworkInterface{
|
||||
&compute.NetworkInterface{
|
||||
AccessConfigs: []*compute.AccessConfig{
|
||||
&compute.AccessConfig{
|
||||
Type: "ONE_TO_ONE_NAT",
|
||||
Name: "External NAT",
|
||||
NatIP: ip,
|
||||
},
|
||||
},
|
||||
Network: cl.projectAPIURL() + "/global/networks/default",
|
||||
},
|
||||
},
|
||||
ServiceAccounts: []*compute.ServiceAccount{
|
||||
{
|
||||
Email: "default",
|
||||
Scopes: cl.Scopes,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
log.Printf("Creating instance...")
|
||||
op, err := cl.computeService.Instances.Insert(cl.GCEProjectID, cl.zone(), instance).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create instance: %v", err)
|
||||
}
|
||||
opName := op.Name
|
||||
log.Printf("Created. Waiting on operation %v", opName)
|
||||
OpLoop:
|
||||
for {
|
||||
time.Sleep(2 * time.Second)
|
||||
op, err := cl.computeService.ZoneOperations.Get(cl.GCEProjectID, cl.zone(), opName).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get op %s: %v", opName, err)
|
||||
}
|
||||
switch op.Status {
|
||||
case "PENDING", "RUNNING":
|
||||
log.Printf("Waiting on operation %v", opName)
|
||||
continue
|
||||
case "DONE":
|
||||
if op.Error != nil {
|
||||
for _, operr := range op.Error.Errors {
|
||||
log.Printf("Error: %+v", operr)
|
||||
}
|
||||
log.Fatalf("Failed to start.")
|
||||
}
|
||||
log.Printf("Success. %+v", op)
|
||||
break OpLoop
|
||||
default:
|
||||
log.Fatalf("Unknown status %q: %+v", op.Status, op)
|
||||
}
|
||||
}
|
||||
|
||||
inst, err = cl.computeService.Instances.Get(cl.GCEProjectID, cl.zone(), cl.instName()).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting instance after creation: %v", err)
|
||||
}
|
||||
ij, _ := json.MarshalIndent(inst, "", " ")
|
||||
log.Printf("%s", ij)
|
||||
log.Printf("Instance created.")
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// returns nil if instance doesn't exist.
|
||||
func (cl *cloudLaunch) lookupInstance() *compute.Instance {
|
||||
inst, err := cl.computeService.Instances.Get(cl.GCEProjectID, cl.zone(), cl.instName()).Do()
|
||||
if ae, ok := err.(*googleapi.Error); ok && ae.Code == 404 {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
log.Fatalf("Instances.Get: %v", err)
|
||||
}
|
||||
return inst
|
||||
}
|
||||
|
||||
func (cl *cloudLaunch) instanceDisk() *compute.AttachedDisk {
|
||||
imageURL, err := gceutil.CoreOSImageURL(cl.oauthClient)
|
||||
if err != nil {
|
||||
log.Fatalf("error looking up latest CoreOS stable image: %v", err)
|
||||
}
|
||||
diskName := cl.instName() + "-coreos-stateless-pd"
|
||||
var diskType string
|
||||
if cl.SSD {
|
||||
diskType = cl.projectAPIURL() + "/zones/" + cl.zone() + "/diskTypes/pd-ssd"
|
||||
}
|
||||
return &compute.AttachedDisk{
|
||||
AutoDelete: true,
|
||||
Boot: true,
|
||||
Type: "PERSISTENT",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
DiskName: diskName,
|
||||
SourceImage: imageURL,
|
||||
DiskSizeGb: 50,
|
||||
DiskType: diskType,
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -1,110 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package gceutil provides utility functions to help with instances on
|
||||
// Google Compute Engine.
|
||||
package gceutil // import "go4.org/cloud/google/gceutil"
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/api/compute/v1"
|
||||
)
|
||||
|
||||
// CoreOSImageURL returns the URL of the latest stable CoreOS image for running on Google Compute Engine.
|
||||
func CoreOSImageURL(cl *http.Client) (string, error) {
|
||||
resp, err := cl.Get("https://www.googleapis.com/compute/v1/projects/coreos-cloud/global/images")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
type coreOSImage struct {
|
||||
SelfLink string
|
||||
CreationTimestamp time.Time
|
||||
Name string
|
||||
}
|
||||
|
||||
type coreOSImageList struct {
|
||||
Items []coreOSImage
|
||||
}
|
||||
|
||||
imageList := &coreOSImageList{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(imageList); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if imageList == nil || len(imageList.Items) == 0 {
|
||||
return "", errors.New("no images list in response")
|
||||
}
|
||||
|
||||
imageURL := ""
|
||||
var max time.Time // latest stable image creation time
|
||||
for _, v := range imageList.Items {
|
||||
if !strings.HasPrefix(v.Name, "coreos-stable") {
|
||||
continue
|
||||
}
|
||||
if v.CreationTimestamp.After(max) {
|
||||
max = v.CreationTimestamp
|
||||
imageURL = v.SelfLink
|
||||
}
|
||||
}
|
||||
if imageURL == "" {
|
||||
return "", errors.New("no stable coreOS image found")
|
||||
}
|
||||
return imageURL, nil
|
||||
}
|
||||
|
||||
// InstanceGroupAndManager contains both an InstanceGroup and
|
||||
// its InstanceGroupManager, if any.
|
||||
type InstanceGroupAndManager struct {
|
||||
Group *compute.InstanceGroup
|
||||
|
||||
// Manager is the manager of the Group. It may be nil.
|
||||
Manager *compute.InstanceGroupManager
|
||||
}
|
||||
|
||||
// InstanceGroups returns all the instance groups in a project's zone, along
|
||||
// with their associated InstanceGroupManagers.
|
||||
// The returned map is keyed by the instance group identifier URL.
|
||||
func InstanceGroups(svc *compute.Service, proj, zone string) (map[string]InstanceGroupAndManager, error) {
|
||||
managerList, err := svc.InstanceGroupManagers.List(proj, zone).Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if managerList.NextPageToken != "" {
|
||||
return nil, errors.New("too many managers; pagination not supported")
|
||||
}
|
||||
managedBy := make(map[string]*compute.InstanceGroupManager) // instance group URL -> its manager
|
||||
for _, it := range managerList.Items {
|
||||
managedBy[it.InstanceGroup] = it
|
||||
}
|
||||
groupList, err := svc.InstanceGroups.List(proj, zone).Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if groupList.NextPageToken != "" {
|
||||
return nil, errors.New("too many instance groups; pagination not supported")
|
||||
}
|
||||
ret := make(map[string]InstanceGroupAndManager)
|
||||
for _, it := range groupList.Items {
|
||||
ret[it.SelfLink] = InstanceGroupAndManager{it, managedBy[it.SelfLink]}
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
@ -1,180 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package gcsutil provides tools for accessing Google Cloud Storage until they can be
|
||||
// completely replaced by cloud.google.com/go/storage.
|
||||
package gcsutil // import "go4.org/cloud/google/gcsutil"
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"go4.org/ctxutil"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const gsAccessURL = "https://storage.googleapis.com"
|
||||
|
||||
// An Object holds the name of an object (its bucket and key) within
|
||||
// Google Cloud Storage.
|
||||
type Object struct {
|
||||
Bucket string
|
||||
Key string
|
||||
}
|
||||
|
||||
func (o *Object) valid() error {
|
||||
if o == nil {
|
||||
return errors.New("invalid nil Object")
|
||||
}
|
||||
if o.Bucket == "" {
|
||||
return errors.New("missing required Bucket field in Object")
|
||||
}
|
||||
if o.Key == "" {
|
||||
return errors.New("missing required Key field in Object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A SizedObject holds the bucket, key, and size of an object.
|
||||
type SizedObject struct {
|
||||
Object
|
||||
Size int64
|
||||
}
|
||||
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil *Object>"
|
||||
}
|
||||
return fmt.Sprintf("%v/%v", o.Bucket, o.Key)
|
||||
}
|
||||
|
||||
func (so SizedObject) String() string {
|
||||
return fmt.Sprintf("%v/%v (%vB)", so.Bucket, so.Key, so.Size)
|
||||
}
|
||||
|
||||
// Makes a simple body-less google storage request
|
||||
func simpleRequest(method, url_ string) (*http.Request, error) {
|
||||
req, err := http.NewRequest(method, url_, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("x-goog-api-version", "2")
|
||||
return req, err
|
||||
}
|
||||
|
||||
// ErrInvalidRange is used when the server has returned http.StatusRequestedRangeNotSatisfiable.
|
||||
var ErrInvalidRange = errors.New("gcsutil: requested range not satisfiable")
|
||||
|
||||
// GetPartialObject fetches part of a Google Cloud Storage object.
|
||||
// This function relies on the ctx ctxutil.HTTPClient value being set to an OAuth2
|
||||
// authorized and authenticated HTTP client.
|
||||
// If length is negative, the rest of the object is returned.
|
||||
// It returns ErrInvalidRange if the server replies with http.StatusRequestedRangeNotSatisfiable.
|
||||
// The caller must call Close on the returned value.
|
||||
func GetPartialObject(ctx context.Context, obj Object, offset, length int64) (io.ReadCloser, error) {
|
||||
if offset < 0 {
|
||||
return nil, errors.New("invalid negative offset")
|
||||
}
|
||||
if err := obj.valid(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := simpleRequest("GET", gsAccessURL+"/"+obj.Bucket+"/"+obj.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if length >= 0 {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
|
||||
} else {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
|
||||
}
|
||||
req.Cancel = ctx.Done()
|
||||
res, err := ctxutil.Client(ctx).Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GET (offset=%d, length=%d) failed: %v\n", offset, length, err)
|
||||
}
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
res.Body.Close()
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
if !(res.StatusCode == http.StatusPartialContent || (offset == 0 && res.StatusCode == http.StatusOK)) {
|
||||
res.Body.Close()
|
||||
if res.StatusCode == http.StatusRequestedRangeNotSatisfiable {
|
||||
return nil, ErrInvalidRange
|
||||
}
|
||||
return nil, fmt.Errorf("GET (offset=%d, length=%d) got failed status: %v\n", offset, length, res.Status)
|
||||
}
|
||||
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
// EnumerateObjects lists the objects in a bucket.
|
||||
// This function relies on the ctx oauth2.HTTPClient value being set to an OAuth2
|
||||
// authorized and authenticated HTTP client.
|
||||
// If after is non-empty, listing will begin with lexically greater object names.
|
||||
// If limit is non-zero, the length of the list will be limited to that number.
|
||||
func EnumerateObjects(ctx context.Context, bucket, after string, limit int) ([]*storage.ObjectAttrs, error) {
|
||||
// Build url, with query params
|
||||
var params []string
|
||||
if after != "" {
|
||||
params = append(params, "marker="+url.QueryEscape(after))
|
||||
}
|
||||
if limit > 0 {
|
||||
params = append(params, fmt.Sprintf("max-keys=%v", limit))
|
||||
}
|
||||
query := ""
|
||||
if len(params) > 0 {
|
||||
query = "?" + strings.Join(params, "&")
|
||||
}
|
||||
|
||||
req, err := simpleRequest("GET", gsAccessURL+"/"+bucket+"/"+query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Cancel = ctx.Done()
|
||||
res, err := ctxutil.Client(ctx).Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("gcsutil: bad enumerate response code: %v", res.Status)
|
||||
}
|
||||
|
||||
var xres struct {
|
||||
Contents []SizedObject
|
||||
}
|
||||
if err = xml.NewDecoder(res.Body).Decode(&xres); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objAttrs := make([]*storage.ObjectAttrs, len(xres.Contents))
|
||||
for k, o := range xres.Contents {
|
||||
objAttrs[k] = &storage.ObjectAttrs{
|
||||
Name: o.Key,
|
||||
Size: o.Size,
|
||||
}
|
||||
}
|
||||
|
||||
return objAttrs, nil
|
||||
}
|
||||
@ -1,44 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package ctxutil contains golang.org/x/net/context related utilities.
|
||||
package ctxutil // import "go4.org/ctxutil"
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
// HTTPClient is the context key to use with golang.org/x/net/context's WithValue function
|
||||
// to associate an *http.Client value with a context.
|
||||
//
|
||||
// We use the same value as the oauth2 package (which first introduced this key) rather
|
||||
// than creating a new one and forcing users to possibly set two.
|
||||
var HTTPClient = oauth2.HTTPClient
|
||||
|
||||
// Client returns the HTTP client to use for the provided context.
|
||||
// If ctx is non-nil and has an associated HTTP client, that client is returned.
|
||||
// Otherwise, http.DefaultClient is returned.
|
||||
func Client(ctx context.Context) *http.Client {
|
||||
if ctx != nil {
|
||||
if hc, ok := ctx.Value(HTTPClient).(*http.Client); ok {
|
||||
return hc
|
||||
}
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
/*
|
||||
Copyright 2011 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package errorutil helps make better error messages.
|
||||
package errorutil // import "go4.org/errorutil"
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// HighlightBytePosition takes a reader and the location in bytes of a parse
|
||||
// error (for instance, from json.SyntaxError.Offset) and returns the line, column,
|
||||
// and pretty-printed context around the error with an arrow indicating the exact
|
||||
// position of the syntax error.
|
||||
func HighlightBytePosition(f io.Reader, pos int64) (line, col int, highlight string) {
|
||||
line = 1
|
||||
br := bufio.NewReader(f)
|
||||
lastLine := ""
|
||||
thisLine := new(bytes.Buffer)
|
||||
for n := int64(0); n < pos; n++ {
|
||||
b, err := br.ReadByte()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if b == '\n' {
|
||||
lastLine = thisLine.String()
|
||||
thisLine.Reset()
|
||||
line++
|
||||
col = 1
|
||||
} else {
|
||||
col++
|
||||
thisLine.WriteByte(b)
|
||||
}
|
||||
}
|
||||
if line > 1 {
|
||||
highlight += fmt.Sprintf("%5d: %s\n", line-1, lastLine)
|
||||
}
|
||||
highlight += fmt.Sprintf("%5d: %s\n", line, thisLine.String())
|
||||
highlight += fmt.Sprintf("%s^\n", strings.Repeat(" ", col+5))
|
||||
return
|
||||
}
|
||||
@ -1,59 +0,0 @@
|
||||
/*
|
||||
Copyright 2014 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package fault handles fault injection for testing.
|
||||
package fault // import "go4.org/fault"
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var fakeErr = errors.New("fake injected error for testing")
|
||||
|
||||
// An Injector reports whether fake errors should be returned.
|
||||
type Injector struct {
|
||||
failPercent int
|
||||
}
|
||||
|
||||
// NewInjector returns a new fault injector with the given name. The
|
||||
// environment variable "FAULT_" + capital(name) + "_FAIL_PERCENT"
|
||||
// controls the percentage of requests that fail. If undefined or
|
||||
// zero, no requests fail.
|
||||
func NewInjector(name string) *Injector {
|
||||
var failPercent, _ = strconv.Atoi(os.Getenv("FAULT_" + strings.ToUpper(name) + "_FAIL_PERCENT"))
|
||||
return &Injector{
|
||||
failPercent: failPercent,
|
||||
}
|
||||
}
|
||||
|
||||
// ShouldFail reports whether a fake error should be returned.
|
||||
func (in *Injector) ShouldFail() bool {
|
||||
return in.failPercent > 0 && in.failPercent > rand.Intn(100)
|
||||
}
|
||||
|
||||
// FailErr checks ShouldFail and, if true, assigns a fake error to err
|
||||
// and returns true.
|
||||
func (in *Injector) FailErr(err *error) bool {
|
||||
if !in.ShouldFail() {
|
||||
return false
|
||||
}
|
||||
*err = fakeErr
|
||||
return true
|
||||
}
|
||||
@ -1,95 +0,0 @@
|
||||
// +build ignore
|
||||
|
||||
/*
|
||||
Copyright 2016 The Go4 Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// The serve_on_cloud program deploys an HTTP server on Google Compute Engine,
|
||||
// serving from Google Cloud Storage. Its purpose is to help testing
|
||||
// go4.org/cloud/cloudlaunch and go4.org/wkfs/gcs.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"go4.org/cloud/cloudlaunch"
|
||||
"go4.org/wkfs"
|
||||
_ "go4.org/wkfs/gcs"
|
||||
|
||||
"cloud.google.com/go/compute/metadata"
|
||||
storageapi "google.golang.org/api/storage/v1"
|
||||
compute "google.golang.org/api/compute/v1"
|
||||
)
|
||||
|
||||
var httpAddr = flag.String("http", ":80", "HTTP address")
|
||||
|
||||
var gcsBucket string
|
||||
|
||||
func serveHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
rc, err := wkfs.Open(path.Join("/gcs", gcsBucket, r.URL.Path))
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("could not open %v: %v", r.URL.Path, err), 500)
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
http.ServeContent(w, r, r.URL.Path, time.Now(), rc)
|
||||
}
|
||||
|
||||
func main() {
|
||||
if !metadata.OnGCE() {
|
||||
bucket := os.Getenv("GCSBUCKET")
|
||||
if bucket == "" {
|
||||
log.Fatal("You need to set the GCSBUCKET env var to specify the Google Cloud Storage bucket to serve from.")
|
||||
}
|
||||
projectID := os.Getenv("GCEPROJECTID")
|
||||
if projectID == "" {
|
||||
log.Fatal("You need to set the GCEPROJECTID env var to specify the Google Cloud project where the instance will run.")
|
||||
}
|
||||
(&cloudlaunch.Config{
|
||||
Name: "serveoncloud",
|
||||
BinaryBucket: bucket,
|
||||
GCEProjectID: projectID,
|
||||
Scopes: []string{
|
||||
storageapi.DevstorageFullControlScope,
|
||||
compute.ComputeScope,
|
||||
},
|
||||
}).MaybeDeploy()
|
||||
return
|
||||
}
|
||||
|
||||
flag.Parse()
|
||||
|
||||
storageURLRxp := regexp.MustCompile(`https://storage.googleapis.com/(.+?)/serveoncloud.*`)
|
||||
cloudConfig, err := metadata.InstanceAttributeValue("user-data")
|
||||
if err != nil || cloudConfig == "" {
|
||||
log.Fatalf("could not get cloud config from metadata: %v", err)
|
||||
}
|
||||
m := storageURLRxp.FindStringSubmatch(cloudConfig)
|
||||
if len(m) < 2 {
|
||||
log.Fatal("storage URL not found in cloud config")
|
||||
}
|
||||
gcsBucket = m[1]
|
||||
|
||||
http.HandleFunc("/", serveHTTP)
|
||||
|
||||
log.Fatal(http.ListenAndServe(*httpAddr, nil))
|
||||
}
|
||||
@ -1,321 +0,0 @@
|
||||
/*
|
||||
Copyright 2011 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package jsonconfig
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"go4.org/errorutil"
|
||||
"go4.org/wkfs"
|
||||
)
|
||||
|
||||
type stringVector struct {
|
||||
v []string
|
||||
}
|
||||
|
||||
func (v *stringVector) Push(s string) {
|
||||
v.v = append(v.v, s)
|
||||
}
|
||||
|
||||
func (v *stringVector) Pop() {
|
||||
v.v = v.v[:len(v.v)-1]
|
||||
}
|
||||
|
||||
func (v *stringVector) Last() string {
|
||||
return v.v[len(v.v)-1]
|
||||
}
|
||||
|
||||
// A File is the type returned by ConfigParser.Open.
|
||||
type File interface {
|
||||
io.ReadSeeker
|
||||
io.Closer
|
||||
Name() string
|
||||
}
|
||||
|
||||
// ConfigParser specifies the environment for parsing a config file
|
||||
// and evaluating expressions.
|
||||
type ConfigParser struct {
|
||||
rootJSON Obj
|
||||
|
||||
touchedFiles map[string]bool
|
||||
includeStack stringVector
|
||||
|
||||
// Open optionally specifies an opener function.
|
||||
Open func(filename string) (File, error)
|
||||
|
||||
// IncludeDirs optionally specifies where to find the other config files which are child
|
||||
// objects of this config, if any. Even if nil, the working directory is always searched
|
||||
// first.
|
||||
IncludeDirs []string
|
||||
}
|
||||
|
||||
func (c *ConfigParser) open(filename string) (File, error) {
|
||||
if c.Open == nil {
|
||||
return wkfs.Open(filename)
|
||||
}
|
||||
return c.Open(filename)
|
||||
}
|
||||
|
||||
// Validates variable names for config _env expresssions
|
||||
var envPattern = regexp.MustCompile(`\$\{[A-Za-z0-9_]+\}`)
|
||||
|
||||
// ReadFile parses the provided path and returns the config file.
|
||||
// If path is empty, the c.Open function must be defined.
|
||||
func (c *ConfigParser) ReadFile(path string) (Obj, error) {
|
||||
if path == "" && c.Open == nil {
|
||||
return nil, errors.New("ReadFile of empty string but Open hook not defined")
|
||||
}
|
||||
c.touchedFiles = make(map[string]bool)
|
||||
var err error
|
||||
c.rootJSON, err = c.recursiveReadJSON(path)
|
||||
return c.rootJSON, err
|
||||
}
|
||||
|
||||
// Decodes and evaluates a json config file, watching for include cycles.
|
||||
func (c *ConfigParser) recursiveReadJSON(configPath string) (decodedObject map[string]interface{}, err error) {
|
||||
if configPath != "" {
|
||||
absConfigPath, err := filepath.Abs(configPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to expand absolute path for %s", configPath)
|
||||
}
|
||||
if c.touchedFiles[absConfigPath] {
|
||||
return nil, fmt.Errorf("ConfigParser include cycle detected reading config: %v",
|
||||
absConfigPath)
|
||||
}
|
||||
c.touchedFiles[absConfigPath] = true
|
||||
|
||||
c.includeStack.Push(absConfigPath)
|
||||
defer c.includeStack.Pop()
|
||||
}
|
||||
|
||||
var f File
|
||||
if f, err = c.open(configPath); err != nil {
|
||||
return nil, fmt.Errorf("Failed to open config: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
decodedObject = make(map[string]interface{})
|
||||
dj := json.NewDecoder(f)
|
||||
if err = dj.Decode(&decodedObject); err != nil {
|
||||
extra := ""
|
||||
if serr, ok := err.(*json.SyntaxError); ok {
|
||||
if _, serr := f.Seek(0, os.SEEK_SET); serr != nil {
|
||||
log.Fatalf("seek error: %v", serr)
|
||||
}
|
||||
line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset)
|
||||
extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
|
||||
line, col, serr.Offset, highlight)
|
||||
}
|
||||
return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v",
|
||||
f.Name(), extra, err)
|
||||
}
|
||||
|
||||
if err = c.evaluateExpressions(decodedObject, nil, false); err != nil {
|
||||
return nil, fmt.Errorf("error expanding JSON config expressions in %s:\n%v",
|
||||
f.Name(), err)
|
||||
}
|
||||
|
||||
return decodedObject, nil
|
||||
}
|
||||
|
||||
var regFunc = map[string]expanderFunc{}
|
||||
|
||||
// RegisterFunc registers a new function that may be called from JSON
|
||||
// configs using an array of the form ["_name", arg0, argN...].
|
||||
// The provided name must begin with an underscore.
|
||||
func RegisterFunc(name string, fn func(c *ConfigParser, v []interface{}) (interface{}, error)) {
|
||||
if len(name) < 2 || !strings.HasPrefix(name, "_") {
|
||||
panic("illegal name")
|
||||
}
|
||||
if _, dup := regFunc[name]; dup {
|
||||
panic("duplicate registration of " + name)
|
||||
}
|
||||
regFunc[name] = fn
|
||||
}
|
||||
|
||||
type expanderFunc func(c *ConfigParser, v []interface{}) (interface{}, error)
|
||||
|
||||
func namedExpander(name string) (fn expanderFunc, ok bool) {
|
||||
switch name {
|
||||
case "_env":
|
||||
return (*ConfigParser).expandEnv, true
|
||||
case "_fileobj":
|
||||
return (*ConfigParser).expandFile, true
|
||||
}
|
||||
fn, ok = regFunc[name]
|
||||
return
|
||||
}
|
||||
|
||||
func (c *ConfigParser) evalValue(v interface{}) (interface{}, error) {
|
||||
sl, ok := v.([]interface{})
|
||||
if !ok {
|
||||
return v, nil
|
||||
}
|
||||
if name, ok := sl[0].(string); ok {
|
||||
if expander, ok := namedExpander(name); ok {
|
||||
newval, err := expander(c, sl[1:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newval, nil
|
||||
}
|
||||
}
|
||||
for i, oldval := range sl {
|
||||
newval, err := c.evalValue(oldval)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sl[i] = newval
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// CheckTypes parses m and returns an error if it encounters a type or value
|
||||
// that is not supported by this package.
|
||||
func (c *ConfigParser) CheckTypes(m map[string]interface{}) error {
|
||||
return c.evaluateExpressions(m, nil, true)
|
||||
}
|
||||
|
||||
// evaluateExpressions parses recursively m, populating it with the values
|
||||
// that are found, unless testOnly is true.
|
||||
func (c *ConfigParser) evaluateExpressions(m map[string]interface{}, seenKeys []string, testOnly bool) error {
|
||||
for k, ei := range m {
|
||||
thisPath := append(seenKeys, k)
|
||||
switch subval := ei.(type) {
|
||||
case string, bool, float64, nil:
|
||||
continue
|
||||
case []interface{}:
|
||||
if len(subval) == 0 {
|
||||
continue
|
||||
}
|
||||
evaled, err := c.evalValue(subval)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: value error %v", strings.Join(thisPath, "."), err)
|
||||
}
|
||||
if !testOnly {
|
||||
m[k] = evaled
|
||||
}
|
||||
case map[string]interface{}:
|
||||
if err := c.evaluateExpressions(subval, thisPath, testOnly); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("%s: unhandled type %T", strings.Join(thisPath, "."), ei)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Permit either:
|
||||
// ["_env", "VARIABLE"] (required to be set)
|
||||
// or ["_env", "VARIABLE", "default_value"]
|
||||
func (c *ConfigParser) expandEnv(v []interface{}) (interface{}, error) {
|
||||
hasDefault := false
|
||||
def := ""
|
||||
if len(v) < 1 || len(v) > 2 {
|
||||
return "", fmt.Errorf("_env expansion expected 1 or 2 args, got %d", len(v))
|
||||
}
|
||||
s, ok := v[0].(string)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("Expected a string after _env expansion; got %#v", v[0])
|
||||
}
|
||||
boolDefault, wantsBool := false, false
|
||||
if len(v) == 2 {
|
||||
hasDefault = true
|
||||
switch vdef := v[1].(type) {
|
||||
case string:
|
||||
def = vdef
|
||||
case bool:
|
||||
wantsBool = true
|
||||
boolDefault = vdef
|
||||
default:
|
||||
return "", fmt.Errorf("Expected default value in %q _env expansion; got %#v", s, v[1])
|
||||
}
|
||||
}
|
||||
var err error
|
||||
expanded := envPattern.ReplaceAllStringFunc(s, func(match string) string {
|
||||
envVar := match[2 : len(match)-1]
|
||||
val := os.Getenv(envVar)
|
||||
// Special case:
|
||||
if val == "" && envVar == "USER" && runtime.GOOS == "windows" {
|
||||
val = os.Getenv("USERNAME")
|
||||
}
|
||||
if val == "" {
|
||||
if hasDefault {
|
||||
return def
|
||||
}
|
||||
err = fmt.Errorf("couldn't expand environment variable %q", envVar)
|
||||
}
|
||||
return val
|
||||
})
|
||||
if wantsBool {
|
||||
if expanded == "" {
|
||||
return boolDefault, nil
|
||||
}
|
||||
return strconv.ParseBool(expanded)
|
||||
}
|
||||
return expanded, err
|
||||
}
|
||||
|
||||
func (c *ConfigParser) expandFile(v []interface{}) (exp interface{}, err error) {
|
||||
if len(v) != 1 {
|
||||
return "", fmt.Errorf("_file expansion expected 1 arg, got %d", len(v))
|
||||
}
|
||||
var incPath string
|
||||
if incPath, err = c.ConfigFilePath(v[0].(string)); err != nil {
|
||||
return "", fmt.Errorf("Included config does not exist: %v", v[0])
|
||||
}
|
||||
if exp, err = c.recursiveReadJSON(incPath); err != nil {
|
||||
return "", fmt.Errorf("In file included from %s:\n%v",
|
||||
c.includeStack.Last(), err)
|
||||
}
|
||||
return exp, nil
|
||||
}
|
||||
|
||||
// ConfigFilePath checks if configFile is found and returns a usable path to it.
|
||||
// It first checks if configFile is an absolute path, or if it's found in the
|
||||
// current working directory. If not, it then checks if configFile is in one of
|
||||
// c.IncludeDirs. It returns an error if configFile is absolute and could not be
|
||||
// statted, or os.ErrNotExist if configFile was not found.
|
||||
func (c *ConfigParser) ConfigFilePath(configFile string) (path string, err error) {
|
||||
// Try to open as absolute / relative to CWD
|
||||
_, err = os.Stat(configFile)
|
||||
if err != nil && filepath.IsAbs(configFile) {
|
||||
return "", err
|
||||
}
|
||||
if err == nil {
|
||||
return configFile, nil
|
||||
}
|
||||
|
||||
for _, d := range c.IncludeDirs {
|
||||
if _, err := os.Stat(filepath.Join(d, configFile)); err == nil {
|
||||
return filepath.Join(d, configFile), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
@ -1,297 +0,0 @@
|
||||
/*
|
||||
Copyright 2011 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package jsonconfig defines a helper type for JSON objects to be
|
||||
// used for configuration.
|
||||
package jsonconfig // import "go4.org/jsonconfig"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Obj is a JSON configuration map.
|
||||
type Obj map[string]interface{}
|
||||
|
||||
// ReadFile reads JSON config data from the specified open file, expanding
|
||||
// all expressions. Use *ConfigParser.ReadFile instead if you
|
||||
// need to set c.IncludeDirs.
|
||||
func ReadFile(configPath string) (Obj, error) {
|
||||
var c ConfigParser
|
||||
return c.ReadFile(configPath)
|
||||
}
|
||||
|
||||
func (jc Obj) RequiredObject(key string) Obj {
|
||||
return jc.obj(key, false)
|
||||
}
|
||||
|
||||
func (jc Obj) OptionalObject(key string) Obj {
|
||||
return jc.obj(key, true)
|
||||
}
|
||||
|
||||
func (jc Obj) obj(key string, optional bool) Obj {
|
||||
jc.noteKnownKey(key)
|
||||
ei, ok := jc[key]
|
||||
if !ok {
|
||||
if optional {
|
||||
return make(Obj)
|
||||
}
|
||||
jc.appendError(fmt.Errorf("Missing required config key %q (object)", key))
|
||||
return make(Obj)
|
||||
}
|
||||
m, ok := ei.(map[string]interface{})
|
||||
if !ok {
|
||||
jc.appendError(fmt.Errorf("Expected config key %q to be an object, not %T", key, ei))
|
||||
return make(Obj)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (jc Obj) RequiredString(key string) string {
|
||||
return jc.string(key, nil)
|
||||
}
|
||||
|
||||
func (jc Obj) OptionalString(key, def string) string {
|
||||
return jc.string(key, &def)
|
||||
}
|
||||
|
||||
func (jc Obj) string(key string, def *string) string {
|
||||
jc.noteKnownKey(key)
|
||||
ei, ok := jc[key]
|
||||
if !ok {
|
||||
if def != nil {
|
||||
return *def
|
||||
}
|
||||
jc.appendError(fmt.Errorf("Missing required config key %q (string)", key))
|
||||
return ""
|
||||
}
|
||||
s, ok := ei.(string)
|
||||
if !ok {
|
||||
jc.appendError(fmt.Errorf("Expected config key %q to be a string", key))
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (jc Obj) RequiredStringOrObject(key string) interface{} {
|
||||
return jc.stringOrObject(key, true)
|
||||
}
|
||||
|
||||
func (jc Obj) OptionalStringOrObject(key string) interface{} {
|
||||
return jc.stringOrObject(key, false)
|
||||
}
|
||||
|
||||
func (jc Obj) stringOrObject(key string, required bool) interface{} {
|
||||
jc.noteKnownKey(key)
|
||||
ei, ok := jc[key]
|
||||
if !ok {
|
||||
if !required {
|
||||
return nil
|
||||
}
|
||||
jc.appendError(fmt.Errorf("Missing required config key %q (string or object)", key))
|
||||
return ""
|
||||
}
|
||||
if _, ok := ei.(map[string]interface{}); ok {
|
||||
return ei
|
||||
}
|
||||
if _, ok := ei.(string); ok {
|
||||
return ei
|
||||
}
|
||||
jc.appendError(fmt.Errorf("Expected config key %q to be a string or object", key))
|
||||
return ""
|
||||
}
|
||||
|
||||
func (jc Obj) RequiredBool(key string) bool {
|
||||
return jc.bool(key, nil)
|
||||
}
|
||||
|
||||
func (jc Obj) OptionalBool(key string, def bool) bool {
|
||||
return jc.bool(key, &def)
|
||||
}
|
||||
|
||||
func (jc Obj) bool(key string, def *bool) bool {
|
||||
jc.noteKnownKey(key)
|
||||
ei, ok := jc[key]
|
||||
if !ok {
|
||||
if def != nil {
|
||||
return *def
|
||||
}
|
||||
jc.appendError(fmt.Errorf("Missing required config key %q (boolean)", key))
|
||||
return false
|
||||
}
|
||||
switch v := ei.(type) {
|
||||
case bool:
|
||||
return v
|
||||
case string:
|
||||
b, err := strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
jc.appendError(fmt.Errorf("Config key %q has bad boolean format %q", key, v))
|
||||
}
|
||||
return b
|
||||
default:
|
||||
jc.appendError(fmt.Errorf("Expected config key %q to be a boolean", key))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (jc Obj) RequiredInt(key string) int {
|
||||
return jc.int(key, nil)
|
||||
}
|
||||
|
||||
func (jc Obj) OptionalInt(key string, def int) int {
|
||||
return jc.int(key, &def)
|
||||
}
|
||||
|
||||
func (jc Obj) int(key string, def *int) int {
|
||||
jc.noteKnownKey(key)
|
||||
ei, ok := jc[key]
|
||||
if !ok {
|
||||
if def != nil {
|
||||
return *def
|
||||
}
|
||||
jc.appendError(fmt.Errorf("Missing required config key %q (integer)", key))
|
||||
return 0
|
||||
}
|
||||
b, ok := ei.(float64)
|
||||
if !ok {
|
||||
jc.appendError(fmt.Errorf("Expected config key %q to be a number", key))
|
||||
return 0
|
||||
}
|
||||
return int(b)
|
||||
}
|
||||
|
||||
func (jc Obj) RequiredInt64(key string) int64 {
|
||||
return jc.int64(key, nil)
|
||||
}
|
||||
|
||||
func (jc Obj) OptionalInt64(key string, def int64) int64 {
|
||||
return jc.int64(key, &def)
|
||||
}
|
||||
|
||||
func (jc Obj) int64(key string, def *int64) int64 {
|
||||
jc.noteKnownKey(key)
|
||||
ei, ok := jc[key]
|
||||
if !ok {
|
||||
if def != nil {
|
||||
return *def
|
||||
}
|
||||
jc.appendError(fmt.Errorf("Missing required config key %q (integer)", key))
|
||||
return 0
|
||||
}
|
||||
b, ok := ei.(float64)
|
||||
if !ok {
|
||||
jc.appendError(fmt.Errorf("Expected config key %q to be a number", key))
|
||||
return 0
|
||||
}
|
||||
return int64(b)
|
||||
}
|
||||
|
||||
func (jc Obj) RequiredList(key string) []string {
|
||||
return jc.requiredList(key, true)
|
||||
}
|
||||
|
||||
func (jc Obj) OptionalList(key string) []string {
|
||||
return jc.requiredList(key, false)
|
||||
}
|
||||
|
||||
func (jc Obj) requiredList(key string, required bool) []string {
|
||||
jc.noteKnownKey(key)
|
||||
ei, ok := jc[key]
|
||||
if !ok {
|
||||
if required {
|
||||
jc.appendError(fmt.Errorf("Missing required config key %q (list of strings)", key))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
eil, ok := ei.([]interface{})
|
||||
if !ok {
|
||||
jc.appendError(fmt.Errorf("Expected config key %q to be a list, not %T", key, ei))
|
||||
return nil
|
||||
}
|
||||
sl := make([]string, len(eil))
|
||||
for i, ei := range eil {
|
||||
s, ok := ei.(string)
|
||||
if !ok {
|
||||
jc.appendError(fmt.Errorf("Expected config key %q index %d to be a string, not %T", key, i, ei))
|
||||
return nil
|
||||
}
|
||||
sl[i] = s
|
||||
}
|
||||
return sl
|
||||
}
|
||||
|
||||
func (jc Obj) noteKnownKey(key string) {
|
||||
_, ok := jc["_knownkeys"]
|
||||
if !ok {
|
||||
jc["_knownkeys"] = make(map[string]bool)
|
||||
}
|
||||
jc["_knownkeys"].(map[string]bool)[key] = true
|
||||
}
|
||||
|
||||
func (jc Obj) appendError(err error) {
|
||||
ei, ok := jc["_errors"]
|
||||
if ok {
|
||||
jc["_errors"] = append(ei.([]error), err)
|
||||
} else {
|
||||
jc["_errors"] = []error{err}
|
||||
}
|
||||
}
|
||||
|
||||
// UnknownKeys returns the keys from the config that have not yet been discovered by one of the RequiredT or OptionalT calls.
|
||||
func (jc Obj) UnknownKeys() []string {
|
||||
ei, ok := jc["_knownkeys"]
|
||||
var known map[string]bool
|
||||
if ok {
|
||||
known = ei.(map[string]bool)
|
||||
}
|
||||
var unknown []string
|
||||
for k, _ := range jc {
|
||||
if ok && known[k] {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, "_") {
|
||||
// Permit keys with a leading underscore as a
|
||||
// form of comments.
|
||||
continue
|
||||
}
|
||||
unknown = append(unknown, k)
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
return unknown
|
||||
}
|
||||
|
||||
func (jc Obj) Validate() error {
|
||||
unknown := jc.UnknownKeys()
|
||||
for _, k := range unknown {
|
||||
jc.appendError(fmt.Errorf("Unknown key %q", k))
|
||||
}
|
||||
|
||||
ei, ok := jc["_errors"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
errList := ei.([]error)
|
||||
if len(errList) == 1 {
|
||||
return errList[0]
|
||||
}
|
||||
strs := make([]string, 0)
|
||||
for _, v := range errList {
|
||||
strs = append(strs, v.Error())
|
||||
}
|
||||
return fmt.Errorf("Multiple errors: " + strings.Join(strs, ", "))
|
||||
}
|
||||
@ -1,114 +0,0 @@
|
||||
/*
|
||||
Copyright 2011 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package jsonconfig
|
||||
|
||||
import (
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testIncludes(configFile string, t *testing.T) {
|
||||
var c ConfigParser
|
||||
c.IncludeDirs = []string{"testdata"}
|
||||
obj, err := c.ReadFile(configFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
two := obj.RequiredObject("two")
|
||||
if err := obj.Validate(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if g, e := two.RequiredString("key"), "value"; g != e {
|
||||
t.Errorf("sub object key = %q; want %q", g, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncludesCWD(t *testing.T) {
|
||||
testIncludes("testdata/include1.json", t)
|
||||
}
|
||||
|
||||
func TestIncludesIncludeDirs(t *testing.T) {
|
||||
testIncludes("testdata/include1bis.json", t)
|
||||
}
|
||||
|
||||
func TestIncludeLoop(t *testing.T) {
|
||||
_, err := ReadFile("testdata/loop1.json")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error about import cycles.")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "include cycle detected") {
|
||||
t.Fatalf("expected an error about import cycles; got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoolEnvs(t *testing.T) {
|
||||
os.Setenv("TEST_EMPTY", "")
|
||||
os.Setenv("TEST_TRUE", "true")
|
||||
os.Setenv("TEST_ONE", "1")
|
||||
os.Setenv("TEST_ZERO", "0")
|
||||
os.Setenv("TEST_FALSE", "false")
|
||||
obj, err := ReadFile("testdata/boolenv.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if str := obj.RequiredString("emptystr"); str != "" {
|
||||
t.Errorf("str = %q, want empty", str)
|
||||
}
|
||||
tests := []struct {
|
||||
key string
|
||||
want bool
|
||||
}{
|
||||
{"def_false", false},
|
||||
{"def_true", true},
|
||||
{"set_true_def_false", true},
|
||||
{"set_false_def_true", false},
|
||||
{"lit_true", true},
|
||||
{"lit_false", false},
|
||||
{"one", true},
|
||||
{"zero", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if v := obj.RequiredBool(tt.key); v != tt.want {
|
||||
t.Errorf("key %q = %v; want %v", tt.key, v, tt.want)
|
||||
}
|
||||
}
|
||||
if err := obj.Validate(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListExpansion(t *testing.T) {
|
||||
os.Setenv("TEST_BAR", "bar")
|
||||
obj, err := ReadFile("testdata/listexpand.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := obj.RequiredString("str")
|
||||
l := obj.RequiredList("list")
|
||||
if err := obj.Validate(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
want := []string{"foo", "bar"}
|
||||
if !reflect.DeepEqual(l, want) {
|
||||
t.Errorf("got = %#v\nwant = %#v", l, want)
|
||||
}
|
||||
if s != "bar" {
|
||||
t.Errorf("str = %q, want %q", s, "bar")
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
{
|
||||
"emptystr": ["_env", "${TEST_EMPTY}", ""],
|
||||
"def_false": ["_env", "${TEST_EMPTY}", false],
|
||||
"def_true": ["_env", "${TEST_EMPTY}", true],
|
||||
"set_true_def_false": ["_env", "${TEST_TRUE}", false],
|
||||
"set_false_def_true": ["_env", "${TEST_FALSE}", true],
|
||||
"one": ["_env", "${TEST_ONE}"],
|
||||
"zero": ["_env", "${TEST_ZERO}"],
|
||||
"lit_true": true,
|
||||
"lit_false": false
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"two": ["_fileobj", "testdata/include2.json"]
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"two": ["_fileobj", "include2.json"]
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"key": "value"
|
||||
}
|
||||
@ -1,4 +0,0 @@
|
||||
{
|
||||
"list": ["foo", ["_env", "${TEST_BAR}"]],
|
||||
"str": ["_env", "${TEST_BAR}"]
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"obj": ["_fileobj", "testdata/loop2.json"]
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"obj": ["_fileobj", "testdata/loop1.json"]
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
/*
|
||||
Copyright 2014 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package legal provides in-process storage for compiled-in licenses.
|
||||
package legal // import "go4.org/legal"
|
||||
|
||||
var licenses []string
|
||||
|
||||
// RegisterLicense stores the license text.
|
||||
// It doesn't check whether the text was already present.
|
||||
func RegisterLicense(text string) {
|
||||
licenses = append(licenses, text)
|
||||
return
|
||||
}
|
||||
|
||||
// Licenses returns a slice of the licenses.
|
||||
func Licenses() []string {
|
||||
return licenses
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
/*
|
||||
Copyright 2014 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package legal
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRegisterLicense(t *testing.T) {
|
||||
initial := len(licenses)
|
||||
RegisterLicense("dummy")
|
||||
if initial+1 != len(licenses) {
|
||||
t.Fatal("didn't add a license")
|
||||
}
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
*~
|
||||
@ -1,186 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Go Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package lock is a file locking library.
|
||||
package lock // import "go4.org/lock"
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Lock locks the given file, creating the file if necessary. If the
|
||||
// file already exists, it must have zero size or an error is returned.
|
||||
// The lock is an exclusive lock (a write lock), but locked files
|
||||
// should neither be read from nor written to. Such files should have
|
||||
// zero size and only exist to co-ordinate ownership across processes.
|
||||
//
|
||||
// A nil Closer is returned if an error occurred. Otherwise, close that
|
||||
// Closer to release the lock.
|
||||
//
|
||||
// On Linux, FreeBSD and OSX, a lock has the same semantics as fcntl(2)'s
|
||||
// advisory locks. In particular, closing any other file descriptor for the
|
||||
// same file will release the lock prematurely.
|
||||
//
|
||||
// Attempting to lock a file that is already locked by the current process
|
||||
// has undefined behavior.
|
||||
//
|
||||
// On other operating systems, lock will fallback to using the presence and
|
||||
// content of a file named name + '.lock' to implement locking behavior.
|
||||
func Lock(name string) (io.Closer, error) {
|
||||
abs, err := filepath.Abs(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lockmu.Lock()
|
||||
defer lockmu.Unlock()
|
||||
if locked[abs] {
|
||||
return nil, fmt.Errorf("file %q already locked", abs)
|
||||
}
|
||||
|
||||
c, err := lockFn(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot acquire lock: %v", err)
|
||||
}
|
||||
locked[abs] = true
|
||||
return c, nil
|
||||
}
|
||||
|
||||
var lockFn = lockPortable
|
||||
|
||||
// lockPortable is a portable version not using fcntl. Doesn't handle crashes as gracefully,
|
||||
// since it can leave stale lock files.
|
||||
func lockPortable(name string) (io.Closer, error) {
|
||||
fi, err := os.Stat(name)
|
||||
if err == nil && fi.Size() > 0 {
|
||||
st := portableLockStatus(name)
|
||||
switch st {
|
||||
case statusLocked:
|
||||
return nil, fmt.Errorf("file %q already locked", name)
|
||||
case statusStale:
|
||||
os.Remove(name)
|
||||
case statusInvalid:
|
||||
return nil, fmt.Errorf("can't Lock file %q: has invalid contents", name)
|
||||
}
|
||||
}
|
||||
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create lock file %s %v", name, err)
|
||||
}
|
||||
if err := json.NewEncoder(f).Encode(&pidLockMeta{OwnerPID: os.Getpid()}); err != nil {
|
||||
return nil, fmt.Errorf("cannot write owner pid: %v", err)
|
||||
}
|
||||
return &unlocker{
|
||||
f: f,
|
||||
abs: name,
|
||||
portable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type lockStatus int
|
||||
|
||||
const (
|
||||
statusInvalid lockStatus = iota
|
||||
statusLocked
|
||||
statusUnlocked
|
||||
statusStale
|
||||
)
|
||||
|
||||
type pidLockMeta struct {
|
||||
OwnerPID int
|
||||
}
|
||||
|
||||
func portableLockStatus(path string) lockStatus {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return statusUnlocked
|
||||
}
|
||||
defer f.Close()
|
||||
var meta pidLockMeta
|
||||
if json.NewDecoder(f).Decode(&meta) != nil {
|
||||
return statusInvalid
|
||||
}
|
||||
if meta.OwnerPID == 0 {
|
||||
return statusInvalid
|
||||
}
|
||||
p, err := os.FindProcess(meta.OwnerPID)
|
||||
if err != nil {
|
||||
// e.g. on Windows
|
||||
return statusStale
|
||||
}
|
||||
// On unix, os.FindProcess always is true, so we have to send
|
||||
// it a signal to see if it's alive.
|
||||
if signalZero != nil {
|
||||
if p.Signal(signalZero) != nil {
|
||||
return statusStale
|
||||
}
|
||||
}
|
||||
return statusLocked
|
||||
}
|
||||
|
||||
var signalZero os.Signal // nil or set by lock_sigzero.go
|
||||
|
||||
var (
|
||||
lockmu sync.Mutex
|
||||
locked = map[string]bool{} // abs path -> true
|
||||
)
|
||||
|
||||
type unlocker struct {
|
||||
portable bool
|
||||
f *os.File
|
||||
abs string
|
||||
// once guards the close method call.
|
||||
once sync.Once
|
||||
// err holds the error returned by Close.
|
||||
err error
|
||||
}
|
||||
|
||||
func (u *unlocker) Close() error {
|
||||
u.once.Do(u.close)
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *unlocker) close() {
|
||||
lockmu.Lock()
|
||||
defer lockmu.Unlock()
|
||||
delete(locked, u.abs)
|
||||
|
||||
if u.portable {
|
||||
// In the portable lock implementation, it's
|
||||
// important to close before removing because
|
||||
// Windows won't allow us to remove an open
|
||||
// file.
|
||||
if err := u.f.Close(); err != nil {
|
||||
u.err = err
|
||||
}
|
||||
if err := os.Remove(u.abs); err != nil {
|
||||
// Note that if both Close and Remove fail,
|
||||
// we care more about the latter than the former
|
||||
// so we'll return that error.
|
||||
u.err = err
|
||||
}
|
||||
return
|
||||
}
|
||||
// In other implementatioons, it's nice for us to clean up.
|
||||
// If we do do this, though, it needs to be before the
|
||||
// u.f.Close below.
|
||||
os.Remove(u.abs)
|
||||
u.err = u.f.Close()
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
// +build appengine
|
||||
|
||||
/*
|
||||
Copyright 2013 The Go Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
func init() {
|
||||
lockFn = lockAppEngine
|
||||
}
|
||||
|
||||
func lockAppEngine(name string) (io.Closer, error) {
|
||||
return nil, errors.New("Lock not available on App Engine")
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Go Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
func init() {
|
||||
lockFn = lockPlan9
|
||||
}
|
||||
|
||||
func lockPlan9(name string) (io.Closer, error) {
|
||||
fi, err := os.Stat(name)
|
||||
if err == nil && fi.Size() > 0 {
|
||||
return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Lock Create of %s failed: %v", name, err)
|
||||
}
|
||||
|
||||
return &unlocker{f: f, abs: name}, nil
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
// +build !appengine
|
||||
// +build linux darwin freebsd openbsd netbsd dragonfly
|
||||
|
||||
/*
|
||||
Copyright 2013 The Go Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import "syscall"
|
||||
|
||||
func init() {
|
||||
signalZero = syscall.Signal(0)
|
||||
}
|
||||
@ -1,222 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Go Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLock(t *testing.T) {
|
||||
testLock(t, false)
|
||||
}
|
||||
|
||||
func TestLockPortable(t *testing.T) {
|
||||
testLock(t, true)
|
||||
}
|
||||
|
||||
func TestLockInChild(t *testing.T) {
|
||||
f := os.Getenv("TEST_LOCK_FILE")
|
||||
if f == "" {
|
||||
// not child
|
||||
return
|
||||
}
|
||||
lock := Lock
|
||||
if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_PORTABLE")); v {
|
||||
lock = lockPortable
|
||||
}
|
||||
|
||||
var lk io.Closer
|
||||
for scan := bufio.NewScanner(os.Stdin); scan.Scan(); {
|
||||
var err error
|
||||
switch scan.Text() {
|
||||
case "lock":
|
||||
lk, err = lock(f)
|
||||
case "unlock":
|
||||
err = lk.Close()
|
||||
lk = nil
|
||||
case "exit":
|
||||
// Simulate a crash, or at least not unlocking the lock.
|
||||
os.Exit(0)
|
||||
default:
|
||||
err = fmt.Errorf("unexpected child command %q", scan.Text())
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
} else {
|
||||
fmt.Println("")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testLock(t *testing.T, portable bool) {
|
||||
lock := Lock
|
||||
if portable {
|
||||
lock = lockPortable
|
||||
}
|
||||
t.Logf("test lock, portable %v", portable)
|
||||
|
||||
td, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(td)
|
||||
|
||||
path := filepath.Join(td, "foo.lock")
|
||||
|
||||
proc := newChildProc(t, path, portable)
|
||||
defer proc.kill()
|
||||
|
||||
t.Logf("First lock in child")
|
||||
if err := proc.do("lock"); err != nil {
|
||||
t.Fatalf("first lock in child process: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Crash child")
|
||||
if err := proc.do("exit"); err != nil {
|
||||
t.Fatalf("crash in child process: %v", err)
|
||||
}
|
||||
|
||||
proc = newChildProc(t, path, portable)
|
||||
defer proc.kill()
|
||||
|
||||
t.Logf("Locking+unlocking in child...")
|
||||
if err := proc.do("lock"); err != nil {
|
||||
t.Fatalf("lock in child process after crashing child: %v", err)
|
||||
}
|
||||
if err := proc.do("unlock"); err != nil {
|
||||
t.Fatalf("lock in child process after crashing child: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Locking in parent...")
|
||||
lk1, err := lock(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Again in parent...")
|
||||
_, err = lock(path)
|
||||
if err == nil {
|
||||
t.Fatal("expected second lock to fail")
|
||||
}
|
||||
|
||||
t.Logf("Locking in child...")
|
||||
if err := proc.do("lock"); err == nil {
|
||||
t.Fatalf("expected lock in child process to fail")
|
||||
}
|
||||
|
||||
t.Logf("Unlocking lock in parent")
|
||||
if err := lk1.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Trying lock again in child...")
|
||||
if err := proc.do("lock"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := proc.do("unlock"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lk3, err := lock(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lk3.Close()
|
||||
}
|
||||
|
||||
type childLockCmd struct {
|
||||
op string
|
||||
reply chan<- error
|
||||
}
|
||||
|
||||
type childProc struct {
|
||||
proc *os.Process
|
||||
c chan childLockCmd
|
||||
}
|
||||
|
||||
func (c *childProc) kill() {
|
||||
c.proc.Kill()
|
||||
}
|
||||
|
||||
func (c *childProc) do(op string) error {
|
||||
reply := make(chan error)
|
||||
c.c <- childLockCmd{
|
||||
op: op,
|
||||
reply: reply,
|
||||
}
|
||||
return <-reply
|
||||
}
|
||||
|
||||
func newChildProc(t *testing.T, path string, portable bool) *childProc {
|
||||
cmd := exec.Command(os.Args[0], "-test.run=LockInChild$")
|
||||
cmd.Env = []string{"TEST_LOCK_FILE=" + path}
|
||||
toChild, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot make pipe: %v", err)
|
||||
}
|
||||
fromChild, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot make pipe: %v", err)
|
||||
}
|
||||
cmd.Stderr = os.Stderr
|
||||
if portable {
|
||||
cmd.Env = append(cmd.Env, "TEST_LOCK_PORTABLE=1")
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("cannot start child: %v", err)
|
||||
}
|
||||
cmdChan := make(chan childLockCmd)
|
||||
go func() {
|
||||
defer fromChild.Close()
|
||||
defer toChild.Close()
|
||||
inScan := bufio.NewScanner(fromChild)
|
||||
for c := range cmdChan {
|
||||
fmt.Fprintln(toChild, c.op)
|
||||
ok := inScan.Scan()
|
||||
if c.op == "exit" {
|
||||
if ok {
|
||||
c.reply <- errors.New("child did not exit")
|
||||
} else {
|
||||
cmd.Wait()
|
||||
c.reply <- nil
|
||||
}
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
panic("child exited early")
|
||||
}
|
||||
if errText := inScan.Text(); errText != "" {
|
||||
c.reply <- errors.New(errText)
|
||||
} else {
|
||||
c.reply <- nil
|
||||
}
|
||||
}
|
||||
}()
|
||||
return &childProc{
|
||||
c: cmdChan,
|
||||
proc: cmd.Process,
|
||||
}
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
// +build linux darwin freebsd openbsd netbsd dragonfly solaris
|
||||
// +build !appengine
|
||||
|
||||
/*
|
||||
Copyright 2013 The Go Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func init() {
|
||||
lockFn = lockFcntl
|
||||
}
|
||||
|
||||
func lockFcntl(name string) (io.Closer, error) {
|
||||
fi, err := os.Stat(name)
|
||||
if err == nil && fi.Size() > 0 {
|
||||
return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
|
||||
}
|
||||
|
||||
f, err := os.Create(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Lock Create of %s failed: %v", name, err)
|
||||
}
|
||||
|
||||
err = unix.FcntlFlock(f.Fd(), unix.F_SETLK, &unix.Flock_t{
|
||||
Type: unix.F_WRLCK,
|
||||
Whence: int16(os.SEEK_SET),
|
||||
Start: 0,
|
||||
Len: 0, // 0 means to lock the entire file.
|
||||
Pid: 0, // only used by F_GETLK
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("Lock FcntlFlock of %s failed: %v", name, err)
|
||||
}
|
||||
return &unlocker{f: f, abs: name}, nil
|
||||
}
|
||||
@ -1,78 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Go Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func init() {
|
||||
lockFn = lockWindows
|
||||
}
|
||||
|
||||
type winUnlocker struct {
|
||||
h windows.Handle
|
||||
abs string
|
||||
// err holds the error returned by Close.
|
||||
err error
|
||||
// once guards the close method call.
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (u *winUnlocker) Close() error {
|
||||
u.once.Do(u.close)
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *winUnlocker) close() {
|
||||
lockmu.Lock()
|
||||
defer lockmu.Unlock()
|
||||
delete(locked, u.abs)
|
||||
|
||||
u.err = windows.CloseHandle(u.h)
|
||||
}
|
||||
|
||||
func lockWindows(name string) (io.Closer, error) {
|
||||
fi, err := os.Stat(name)
|
||||
if err == nil && fi.Size() > 0 {
|
||||
return nil, fmt.Errorf("can't lock file %q: %s", name, "has non-zero size")
|
||||
}
|
||||
|
||||
handle, err := winCreateEphemeral(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creation of lock %s failed: %v", name, err)
|
||||
}
|
||||
|
||||
return &winUnlocker{h: handle, abs: name}, nil
|
||||
}
|
||||
|
||||
func winCreateEphemeral(name string) (windows.Handle, error) {
|
||||
const (
|
||||
FILE_ATTRIBUTE_TEMPORARY = 0x100
|
||||
FILE_FLAG_DELETE_ON_CLOSE = 0x04000000
|
||||
)
|
||||
handle, err := windows.CreateFile(windows.StringToUTF16Ptr(name), 0, 0, nil, windows.OPEN_ALWAYS, FILE_ATTRIBUTE_TEMPORARY|FILE_FLAG_DELETE_ON_CLOSE, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return handle, nil
|
||||
}
|
||||
@ -1,833 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package bmff reads ISO BMFF boxes, as used by HEIF, etc.
|
||||
//
|
||||
// This is not so much as a generic BMFF reader as it is a BMFF reader
|
||||
// as needed by HEIF, though that may change in time. For now, only
|
||||
// boxes necessary for the go4.org/media/heif package have explicit
|
||||
// parsers.
|
||||
//
|
||||
// This package makes no API compatibility promises; it exists
|
||||
// primarily for use by the go4.org/media/heif package.
|
||||
package bmff
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func NewReader(r io.Reader) *Reader {
|
||||
br, ok := r.(*bufio.Reader)
|
||||
if !ok {
|
||||
br = bufio.NewReader(r)
|
||||
}
|
||||
return &Reader{br: bufReader{Reader: br}}
|
||||
}
|
||||
|
||||
type Reader struct {
|
||||
br bufReader
|
||||
lastBox Box // or nil
|
||||
noMoreBoxes bool // a box with size 0 (the final box) was seen
|
||||
}
|
||||
|
||||
type BoxType [4]byte
|
||||
|
||||
// Common box types.
|
||||
var (
|
||||
TypeFtyp = BoxType{'f', 't', 'y', 'p'}
|
||||
TypeMeta = BoxType{'m', 'e', 't', 'a'}
|
||||
)
|
||||
|
||||
func (t BoxType) String() string { return string(t[:]) }
|
||||
|
||||
func (t BoxType) EqualString(s string) bool {
|
||||
// Could be cleaner, but see ohttps://github.com/golang/go/issues/24765
|
||||
return len(s) == 4 && s[0] == t[0] && s[1] == t[1] && s[2] == t[2] && s[3] == t[3]
|
||||
}
|
||||
|
||||
type parseFunc func(b box, br *bufio.Reader) (Box, error)
|
||||
|
||||
// Box represents a BMFF box.
|
||||
type Box interface {
|
||||
Size() int64 // 0 means unknown (will read to end of file)
|
||||
Type() BoxType
|
||||
|
||||
// Parses parses the box, populating the fields
|
||||
// in the returned concrete type.
|
||||
//
|
||||
// If Parse has already been called, Parse returns nil.
|
||||
// If the box type is unknown, the returned error is ErrUnknownBox
|
||||
// and it's guaranteed that no bytes have been read from the box.
|
||||
Parse() (Box, error)
|
||||
|
||||
// Body returns the inner bytes of the box, ignoring the header.
|
||||
// The body may start with the 4 byte header of a "Full Box" if the
|
||||
// box's type derives from a full box. Most users will use Parse
|
||||
// instead.
|
||||
// Body will return a new reader at the beginning of the box if the
|
||||
// outer box has already been parsed.
|
||||
Body() io.Reader
|
||||
}
|
||||
|
||||
// ErrUnknownBox is returned by Box.Parse for unrecognized box types.
|
||||
var ErrUnknownBox = errors.New("heif: unknown box")
|
||||
|
||||
type parserFunc func(b *box, br *bufReader) (Box, error)
|
||||
|
||||
func boxType(s string) BoxType {
|
||||
if len(s) != 4 {
|
||||
panic("bogus boxType length")
|
||||
}
|
||||
return BoxType{s[0], s[1], s[2], s[3]}
|
||||
}
|
||||
|
||||
var parsers = map[BoxType]parserFunc{
|
||||
boxType("dinf"): parseDataInformationBox,
|
||||
boxType("dref"): parseDataReferenceBox,
|
||||
boxType("ftyp"): parseFileTypeBox,
|
||||
boxType("hdlr"): parseHandlerBox,
|
||||
boxType("iinf"): parseItemInfoBox,
|
||||
boxType("infe"): parseItemInfoEntry,
|
||||
boxType("iloc"): parseItemLocationBox,
|
||||
boxType("ipco"): parseItemPropertyContainerBox,
|
||||
boxType("ipma"): parseItemPropertyAssociation,
|
||||
boxType("iprp"): parseItemPropertiesBox,
|
||||
boxType("irot"): parseImageRotation,
|
||||
boxType("ispe"): parseImageSpatialExtentsProperty,
|
||||
boxType("meta"): parseMetaBox,
|
||||
boxType("pitm"): parsePrimaryItemBox,
|
||||
}
|
||||
|
||||
type box struct {
|
||||
size int64 // 0 means unknown, will read to end of file (box container)
|
||||
boxType BoxType
|
||||
body io.Reader
|
||||
parsed Box // if non-nil, the Parsed result
|
||||
slurp []byte // if non-nil, the contents slurped to memory
|
||||
}
|
||||
|
||||
func (b *box) Size() int64 { return b.size }
|
||||
func (b *box) Type() BoxType { return b.boxType }
|
||||
|
||||
func (b *box) Body() io.Reader {
|
||||
if b.slurp != nil {
|
||||
return bytes.NewReader(b.slurp)
|
||||
}
|
||||
return b.body
|
||||
}
|
||||
|
||||
func (b *box) Parse() (Box, error) {
|
||||
if b.parsed != nil {
|
||||
return b.parsed, nil
|
||||
}
|
||||
parser, ok := parsers[b.Type()]
|
||||
if !ok {
|
||||
return nil, ErrUnknownBox
|
||||
}
|
||||
v, err := parser(b, &bufReader{Reader: bufio.NewReader(b.Body())})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.parsed = v
|
||||
return v, nil
|
||||
}
|
||||
|
||||
type FullBox struct {
|
||||
*box
|
||||
Version uint8
|
||||
Flags uint32 // 24 bits
|
||||
}
|
||||
|
||||
// ReadBox reads the next box.
|
||||
//
|
||||
// If the previously read box was not read to completion, ReadBox consumes
|
||||
// the rest of its data.
|
||||
//
|
||||
// At the end, the error is io.EOF.
|
||||
func (r *Reader) ReadBox() (Box, error) {
|
||||
if r.noMoreBoxes {
|
||||
return nil, io.EOF
|
||||
}
|
||||
if r.lastBox != nil {
|
||||
if _, err := io.Copy(ioutil.Discard, r.lastBox.Body()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var buf [8]byte
|
||||
|
||||
_, err := io.ReadFull(r.br, buf[:4])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
box := &box{
|
||||
size: int64(binary.BigEndian.Uint32(buf[:4])),
|
||||
}
|
||||
|
||||
_, err = io.ReadFull(r.br, box.boxType[:]) // 4 more bytes
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Special cases for size:
|
||||
var remain int64
|
||||
switch box.size {
|
||||
case 1:
|
||||
// 1 means it's actually a 64-bit size, after the type.
|
||||
_, err = io.ReadFull(r.br, buf[:8])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
box.size = int64(binary.BigEndian.Uint64(buf[:8]))
|
||||
if box.size < 0 {
|
||||
// Go uses int64 for sizes typically, but BMFF uses uint64.
|
||||
// We assume for now that nobody actually uses boxes larger
|
||||
// than int64.
|
||||
return nil, fmt.Errorf("unexpectedly large box %q", box.boxType)
|
||||
}
|
||||
remain = box.size - 2*4 - 8
|
||||
case 0:
|
||||
// 0 means unknown & to read to end of file. No more boxes.
|
||||
r.noMoreBoxes = true
|
||||
default:
|
||||
remain = box.size - 2*4
|
||||
}
|
||||
if remain < 0 {
|
||||
return nil, fmt.Errorf("Box header for %q has size %d, suggesting %d (negative) bytes remain", box.boxType, box.size, remain)
|
||||
}
|
||||
if box.size > 0 {
|
||||
box.body = io.LimitReader(r.br, remain)
|
||||
} else {
|
||||
box.body = r.br
|
||||
}
|
||||
r.lastBox = box
|
||||
return box, nil
|
||||
}
|
||||
|
||||
// ReadAndParseBox wraps the ReadBox method, ensuring that the read box is of type typ
|
||||
// and parses successfully. It returns the parsed box.
|
||||
func (r *Reader) ReadAndParseBox(typ BoxType) (Box, error) {
|
||||
box, err := r.ReadBox()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading %q box: %v", typ, err)
|
||||
}
|
||||
if box.Type() != typ {
|
||||
return nil, fmt.Errorf("error reading %q box: got box type %q instead", typ, box.Type())
|
||||
}
|
||||
pbox, err := box.Parse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing read %q box: %v", typ, err)
|
||||
}
|
||||
return pbox, nil
|
||||
}
|
||||
|
||||
func readFullBox(outer *box, br *bufReader) (fb FullBox, err error) {
|
||||
fb.box = outer
|
||||
// Parse FullBox header.
|
||||
buf, err := br.Peek(4)
|
||||
if err != nil {
|
||||
return FullBox{}, fmt.Errorf("failed to read 4 bytes of FullBox: %v", err)
|
||||
}
|
||||
fb.Version = buf[0]
|
||||
buf[0] = 0
|
||||
fb.Flags = binary.BigEndian.Uint32(buf[:4])
|
||||
br.Discard(4)
|
||||
return fb, nil
|
||||
}
|
||||
|
||||
type FileTypeBox struct {
|
||||
*box
|
||||
MajorBrand string // 4 bytes
|
||||
MinorVersion string // 4 bytes
|
||||
Compatible []string // all 4 bytes
|
||||
}
|
||||
|
||||
func parseFileTypeBox(outer *box, br *bufReader) (Box, error) {
|
||||
buf, err := br.Peek(8)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ft := &FileTypeBox{
|
||||
box: outer,
|
||||
MajorBrand: string(buf[:4]),
|
||||
MinorVersion: string(buf[4:8]),
|
||||
}
|
||||
br.Discard(8)
|
||||
for {
|
||||
buf, err := br.Peek(4)
|
||||
if err == io.EOF {
|
||||
return ft, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ft.Compatible = append(ft.Compatible, string(buf[:4]))
|
||||
br.Discard(4)
|
||||
}
|
||||
}
|
||||
|
||||
type MetaBox struct {
|
||||
FullBox
|
||||
Children []Box
|
||||
}
|
||||
|
||||
func parseMetaBox(outer *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(outer, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mb := &MetaBox{FullBox: fb}
|
||||
return mb, br.parseAppendBoxes(&mb.Children)
|
||||
}
|
||||
|
||||
func (br *bufReader) parseAppendBoxes(dst *[]Box) error {
|
||||
if br.err != nil {
|
||||
return br.err
|
||||
}
|
||||
boxr := NewReader(br.Reader)
|
||||
for {
|
||||
inner, err := boxr.ReadBox()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
br.err = err
|
||||
return err
|
||||
}
|
||||
slurp, err := ioutil.ReadAll(inner.Body())
|
||||
if err != nil {
|
||||
br.err = err
|
||||
return err
|
||||
}
|
||||
inner.(*box).slurp = slurp
|
||||
*dst = append(*dst, inner)
|
||||
}
|
||||
}
|
||||
|
||||
// ItemInfoEntry represents an "infe" box.
|
||||
//
|
||||
// TODO: currently only parses Version 2 boxes.
|
||||
type ItemInfoEntry struct {
|
||||
FullBox
|
||||
|
||||
ItemID uint16
|
||||
ProtectionIndex uint16
|
||||
ItemType string // always 4 bytes
|
||||
|
||||
Name string
|
||||
|
||||
// If Type == "mime":
|
||||
ContentType string
|
||||
ContentEncoding string
|
||||
|
||||
// If Type == "uri ":
|
||||
ItemURIType string
|
||||
}
|
||||
|
||||
func parseItemInfoEntry(outer *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(outer, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ie := &ItemInfoEntry{FullBox: fb}
|
||||
if fb.Version != 2 {
|
||||
return nil, fmt.Errorf("TODO: found version %d infe box. Only 2 is supported now.", fb.Version)
|
||||
}
|
||||
|
||||
ie.ItemID, _ = br.readUint16()
|
||||
ie.ProtectionIndex, _ = br.readUint16()
|
||||
if !br.ok() {
|
||||
return nil, br.err
|
||||
}
|
||||
buf, err := br.Peek(4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ie.ItemType = string(buf[:4])
|
||||
ie.Name, _ = br.readString()
|
||||
|
||||
switch ie.ItemType {
|
||||
case "mime":
|
||||
ie.ContentType, _ = br.readString()
|
||||
if br.anyRemain() {
|
||||
ie.ContentEncoding, _ = br.readString()
|
||||
}
|
||||
case "uri ":
|
||||
ie.ItemURIType, _ = br.readString()
|
||||
}
|
||||
if !br.ok() {
|
||||
return nil, br.err
|
||||
}
|
||||
return ie, nil
|
||||
}
|
||||
|
||||
// ItemInfoBox represents an "iinf" box.
|
||||
type ItemInfoBox struct {
|
||||
FullBox
|
||||
Count uint16
|
||||
ItemInfos []*ItemInfoEntry
|
||||
}
|
||||
|
||||
func parseItemInfoBox(outer *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(outer, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ib := &ItemInfoBox{FullBox: fb}
|
||||
|
||||
ib.Count, _ = br.readUint16()
|
||||
|
||||
var itemInfos []Box
|
||||
br.parseAppendBoxes(&itemInfos)
|
||||
if br.ok() {
|
||||
for _, box := range itemInfos {
|
||||
pb, err := box.Parse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing ItemInfoEntry in ItemInfoBox: %v", err)
|
||||
}
|
||||
if iie, ok := pb.(*ItemInfoEntry); ok {
|
||||
ib.ItemInfos = append(ib.ItemInfos, iie)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !br.ok() {
|
||||
return FullBox{}, br.err
|
||||
}
|
||||
return ib, nil
|
||||
}
|
||||
|
||||
// bufReader adds some HEIF/BMFF-specific methods around a *bufio.Reader.
|
||||
type bufReader struct {
|
||||
*bufio.Reader
|
||||
err error // sticky error
|
||||
}
|
||||
|
||||
// ok reports whether all previous reads have been error-free.
|
||||
func (br *bufReader) ok() bool { return br.err == nil }
|
||||
|
||||
func (br *bufReader) anyRemain() bool {
|
||||
if br.err != nil {
|
||||
return false
|
||||
}
|
||||
_, err := br.Peek(1)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (br *bufReader) readUintN(bits uint8) (uint64, error) {
|
||||
if br.err != nil {
|
||||
return 0, br.err
|
||||
}
|
||||
if bits == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
nbyte := bits / 8
|
||||
buf, err := br.Peek(int(nbyte))
|
||||
if err != nil {
|
||||
br.err = err
|
||||
return 0, err
|
||||
}
|
||||
defer br.Discard(int(nbyte))
|
||||
switch bits {
|
||||
case 8:
|
||||
return uint64(buf[0]), nil
|
||||
case 16:
|
||||
return uint64(binary.BigEndian.Uint16(buf[:2])), nil
|
||||
case 32:
|
||||
return uint64(binary.BigEndian.Uint32(buf[:4])), nil
|
||||
case 64:
|
||||
return binary.BigEndian.Uint64(buf[:8]), nil
|
||||
default:
|
||||
br.err = fmt.Errorf("invalid uintn read size")
|
||||
return 0, br.err
|
||||
}
|
||||
}
|
||||
|
||||
func (br *bufReader) readUint8() (uint8, error) {
|
||||
if br.err != nil {
|
||||
return 0, br.err
|
||||
}
|
||||
v, err := br.ReadByte()
|
||||
if err != nil {
|
||||
br.err = err
|
||||
return 0, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (br *bufReader) readUint16() (uint16, error) {
|
||||
if br.err != nil {
|
||||
return 0, br.err
|
||||
}
|
||||
buf, err := br.Peek(2)
|
||||
if err != nil {
|
||||
br.err = err
|
||||
return 0, err
|
||||
}
|
||||
v := binary.BigEndian.Uint16(buf[:2])
|
||||
br.Discard(2)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (br *bufReader) readUint32() (uint32, error) {
|
||||
if br.err != nil {
|
||||
return 0, br.err
|
||||
}
|
||||
buf, err := br.Peek(4)
|
||||
if err != nil {
|
||||
br.err = err
|
||||
return 0, err
|
||||
}
|
||||
v := binary.BigEndian.Uint32(buf[:4])
|
||||
br.Discard(4)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (br *bufReader) readString() (string, error) {
|
||||
if br.err != nil {
|
||||
return "", br.err
|
||||
}
|
||||
s0, err := br.ReadString(0)
|
||||
if err != nil {
|
||||
br.err = err
|
||||
return "", err
|
||||
}
|
||||
s := strings.TrimSuffix(s0, "\x00")
|
||||
if len(s) == len(s0) {
|
||||
err = fmt.Errorf("unexpected non-null terminated string")
|
||||
br.err = err
|
||||
return "", err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// HEIF: ipco
|
||||
type ItemPropertyContainerBox struct {
|
||||
*box
|
||||
Properties []Box // of ItemProperty or ItemFullProperty
|
||||
}
|
||||
|
||||
func parseItemPropertyContainerBox(outer *box, br *bufReader) (Box, error) {
|
||||
ipc := &ItemPropertyContainerBox{box: outer}
|
||||
return ipc, br.parseAppendBoxes(&ipc.Properties)
|
||||
}
|
||||
|
||||
// HEIF: iprp
|
||||
type ItemPropertiesBox struct {
|
||||
*box
|
||||
PropertyContainer *ItemPropertyContainerBox
|
||||
Associations []*ItemPropertyAssociation // at least 1
|
||||
}
|
||||
|
||||
func parseItemPropertiesBox(outer *box, br *bufReader) (Box, error) {
|
||||
ip := &ItemPropertiesBox{
|
||||
box: outer,
|
||||
}
|
||||
|
||||
var boxes []Box
|
||||
err := br.parseAppendBoxes(&boxes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(boxes) < 2 {
|
||||
return nil, fmt.Errorf("expect at least 2 boxes in children; got 0")
|
||||
}
|
||||
|
||||
cb, err := boxes[0].Parse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse first box, %q: %v", boxes[0].Type(), err)
|
||||
}
|
||||
|
||||
var ok bool
|
||||
ip.PropertyContainer, ok = cb.(*ItemPropertyContainerBox)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected type %T for ItemPropertieBox.PropertyContainer", cb)
|
||||
}
|
||||
|
||||
// Association boxes
|
||||
ip.Associations = make([]*ItemPropertyAssociation, 0, len(boxes)-1)
|
||||
for _, box := range boxes[1:] {
|
||||
boxp, err := box.Parse()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse association box: %v", err)
|
||||
}
|
||||
ipa, ok := boxp.(*ItemPropertyAssociation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected box %q instead of ItemPropertyAssociation", boxp.Type())
|
||||
}
|
||||
ip.Associations = append(ip.Associations, ipa)
|
||||
}
|
||||
return ip, nil
|
||||
}
|
||||
|
||||
type ItemPropertyAssociation struct {
|
||||
FullBox
|
||||
EntryCount uint32
|
||||
Entries []ItemPropertyAssociationItem
|
||||
}
|
||||
|
||||
// not a box
|
||||
type ItemProperty struct {
|
||||
Essential bool
|
||||
Index uint16
|
||||
}
|
||||
|
||||
// not a box
|
||||
type ItemPropertyAssociationItem struct {
|
||||
ItemID uint32
|
||||
AssociationsCount int // as declared
|
||||
Associations []ItemProperty // as parsed
|
||||
}
|
||||
|
||||
func parseItemPropertyAssociation(outer *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(outer, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ipa := &ItemPropertyAssociation{FullBox: fb}
|
||||
count, _ := br.readUint32()
|
||||
ipa.EntryCount = count
|
||||
|
||||
for i := uint64(0); i < uint64(count) && br.ok(); i++ {
|
||||
var itemID uint32
|
||||
if fb.Version < 1 {
|
||||
itemID16, _ := br.readUint16()
|
||||
itemID = uint32(itemID16)
|
||||
} else {
|
||||
itemID, _ = br.readUint32()
|
||||
}
|
||||
assocCount, _ := br.readUint8()
|
||||
ipai := ItemPropertyAssociationItem{
|
||||
ItemID: itemID,
|
||||
AssociationsCount: int(assocCount),
|
||||
}
|
||||
for j := 0; j < int(assocCount) && br.ok(); j++ {
|
||||
first, _ := br.readUint8()
|
||||
essential := first&(1<<7) != 0
|
||||
first &^= byte(1 << 7)
|
||||
|
||||
var index uint16
|
||||
if fb.Flags&1 != 0 {
|
||||
second, _ := br.readUint8()
|
||||
index = uint16(first)<<8 | uint16(second)
|
||||
} else {
|
||||
index = uint16(first)
|
||||
}
|
||||
ipai.Associations = append(ipai.Associations, ItemProperty{
|
||||
Essential: essential,
|
||||
Index: index,
|
||||
})
|
||||
}
|
||||
ipa.Entries = append(ipa.Entries, ipai)
|
||||
}
|
||||
if !br.ok() {
|
||||
return nil, br.err
|
||||
}
|
||||
return ipa, nil
|
||||
}
|
||||
|
||||
type ImageSpatialExtentsProperty struct {
|
||||
FullBox
|
||||
ImageWidth uint32
|
||||
ImageHeight uint32
|
||||
}
|
||||
|
||||
func parseImageSpatialExtentsProperty(outer *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(outer, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w, err := br.readUint32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h, err := br.readUint32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ImageSpatialExtentsProperty{
|
||||
FullBox: fb,
|
||||
ImageWidth: w,
|
||||
ImageHeight: h,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type OffsetLength struct {
|
||||
Offset, Length uint64
|
||||
}
|
||||
|
||||
// not a box
|
||||
type ItemLocationBoxEntry struct {
|
||||
ItemID uint16
|
||||
ConstructionMethod uint8 // actually uint4
|
||||
DataReferenceIndex uint16
|
||||
BaseOffset uint64 // uint32 or uint64, depending on encoding
|
||||
ExtentCount uint16
|
||||
Extents []OffsetLength
|
||||
}
|
||||
|
||||
// box "iloc"
|
||||
type ItemLocationBox struct {
|
||||
FullBox
|
||||
|
||||
offsetSize, lengthSize, baseOffsetSize, indexSize uint8 // actually uint4
|
||||
|
||||
ItemCount uint16
|
||||
Items []ItemLocationBoxEntry
|
||||
}
|
||||
|
||||
func parseItemLocationBox(outer *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(outer, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ilb := &ItemLocationBox{
|
||||
FullBox: fb,
|
||||
}
|
||||
buf, err := br.Peek(4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ilb.offsetSize = buf[0] >> 4
|
||||
ilb.lengthSize = buf[0] & 15
|
||||
ilb.baseOffsetSize = buf[1] >> 4
|
||||
if fb.Version > 0 { // version 1
|
||||
ilb.indexSize = buf[1] & 15
|
||||
}
|
||||
|
||||
ilb.ItemCount = binary.BigEndian.Uint16(buf[2:4])
|
||||
br.Discard(4)
|
||||
|
||||
for i := 0; br.ok() && i < int(ilb.ItemCount); i++ {
|
||||
var ent ItemLocationBoxEntry
|
||||
ent.ItemID, _ = br.readUint16()
|
||||
if fb.Version > 0 { // version 1
|
||||
cmeth, _ := br.readUint16()
|
||||
ent.ConstructionMethod = byte(cmeth & 15)
|
||||
}
|
||||
ent.DataReferenceIndex, _ = br.readUint16()
|
||||
if br.ok() && ilb.baseOffsetSize > 0 {
|
||||
br.Discard(int(ilb.baseOffsetSize) / 8)
|
||||
}
|
||||
ent.ExtentCount, _ = br.readUint16()
|
||||
for j := 0; br.ok() && j < int(ent.ExtentCount); j++ {
|
||||
var ol OffsetLength
|
||||
ol.Offset, _ = br.readUintN(ilb.offsetSize * 8)
|
||||
ol.Length, _ = br.readUintN(ilb.lengthSize * 8)
|
||||
if br.err != nil {
|
||||
return nil, br.err
|
||||
}
|
||||
ent.Extents = append(ent.Extents, ol)
|
||||
}
|
||||
ilb.Items = append(ilb.Items, ent)
|
||||
}
|
||||
if !br.ok() {
|
||||
return nil, br.err
|
||||
}
|
||||
return ilb, nil
|
||||
}
|
||||
|
||||
// a "hdlr" box.
|
||||
type HandlerBox struct {
|
||||
FullBox
|
||||
HandlerType string // always 4 bytes; usually "pict" for iOS Camera images
|
||||
Name string
|
||||
}
|
||||
|
||||
func parseHandlerBox(gen *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(gen, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hb := &HandlerBox{
|
||||
FullBox: fb,
|
||||
}
|
||||
buf, err := br.Peek(20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hb.HandlerType = string(buf[4:8])
|
||||
br.Discard(20)
|
||||
|
||||
hb.Name, _ = br.readString()
|
||||
return hb, br.err
|
||||
}
|
||||
|
||||
// a "dinf" box
|
||||
type DataInformationBox struct {
|
||||
*box
|
||||
Children []Box
|
||||
}
|
||||
|
||||
func parseDataInformationBox(gen *box, br *bufReader) (Box, error) {
|
||||
dib := &DataInformationBox{box: gen}
|
||||
return dib, br.parseAppendBoxes(&dib.Children)
|
||||
}
|
||||
|
||||
// a "dref" box.
|
||||
type DataReferenceBox struct {
|
||||
FullBox
|
||||
EntryCount uint32
|
||||
Children []Box
|
||||
}
|
||||
|
||||
func parseDataReferenceBox(gen *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(gen, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
drb := &DataReferenceBox{FullBox: fb}
|
||||
drb.EntryCount, _ = br.readUint32()
|
||||
return drb, br.parseAppendBoxes(&drb.Children)
|
||||
}
|
||||
|
||||
// "pitm" box
|
||||
type PrimaryItemBox struct {
|
||||
FullBox
|
||||
ItemID uint16
|
||||
}
|
||||
|
||||
func parsePrimaryItemBox(gen *box, br *bufReader) (Box, error) {
|
||||
fb, err := readFullBox(gen, br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pib := &PrimaryItemBox{FullBox: fb}
|
||||
pib.ItemID, _ = br.readUint16()
|
||||
if !br.ok() {
|
||||
return nil, br.err
|
||||
}
|
||||
return pib, nil
|
||||
}
|
||||
|
||||
// ImageRotation is a HEIF "irot" rotation property.
|
||||
type ImageRotation struct {
|
||||
*box
|
||||
Angle uint8 // 1 means 90 degrees counter-clockwise, 2 means 180 counter-clockwise
|
||||
}
|
||||
|
||||
func parseImageRotation(gen *box, br *bufReader) (Box, error) {
|
||||
v, err := br.readUint8()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ImageRotation{box: gen, Angle: v & 3}, nil
|
||||
}
|
||||
@ -1,200 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// The dumpheif program dumps the structure and metadata of a HEIF file.
|
||||
//
|
||||
// It exists purely for debugging the go4.org/media/heif and
|
||||
// go4.org/media/heif/bmff packages; it makes no backwards
|
||||
// compatibility promises.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/rwcarlsen/goexif/exif"
|
||||
"github.com/rwcarlsen/goexif/tiff"
|
||||
|
||||
"go4.org/media/heif"
|
||||
"go4.org/media/heif/bmff"
|
||||
)
|
||||
|
||||
var (
|
||||
exifItemID uint16
|
||||
exifLoc bmff.ItemLocationBoxEntry
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if flag.NArg() != 1 {
|
||||
fmt.Fprintf(os.Stderr, "usage: dumpheif <file>\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
f, err := os.Open(flag.Arg(0))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
hf := heif.Open(f)
|
||||
|
||||
it, err := hf.PrimaryItem()
|
||||
if err != nil {
|
||||
log.Fatalf("PrimaryItem: %v", err)
|
||||
}
|
||||
fmt.Printf("primary item: %v\n", it.ID)
|
||||
|
||||
width, height, ok := it.SpatialExtents()
|
||||
if ok {
|
||||
fmt.Printf("spatial extents: %d x %d\n", width, height)
|
||||
}
|
||||
fmt.Printf("properties:\n")
|
||||
for _, prop := range it.Properties {
|
||||
fmt.Printf("\t%q: %#v\n", prop.Type(), prop)
|
||||
}
|
||||
if len(it.Properties) == 0 {
|
||||
fmt.Printf("\t(no properties)\n")
|
||||
}
|
||||
|
||||
if ex, err := hf.EXIF(); err == nil {
|
||||
fmt.Printf("EXIF dump:\n")
|
||||
ex, err := exif.Decode(bytes.NewReader(ex))
|
||||
if err != nil {
|
||||
log.Fatalf("EXIF decode: %v", err)
|
||||
}
|
||||
ex.Walk(exifWalkFunc(func(name exif.FieldName, tag *tiff.Tag) error {
|
||||
fmt.Printf("\t%v = %v\n", name, tag)
|
||||
return nil
|
||||
}))
|
||||
fmt.Printf("\n")
|
||||
}
|
||||
|
||||
fmt.Printf("BMFF boxes:\n")
|
||||
r := bmff.NewReader(f)
|
||||
for {
|
||||
box, err := r.ReadBox()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("ReadBox: %v", err)
|
||||
}
|
||||
dumpBox(box, 0)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type exifWalkFunc func(exif.FieldName, *tiff.Tag) error
|
||||
|
||||
func (f exifWalkFunc) Walk(name exif.FieldName, tag *tiff.Tag) error {
|
||||
return f(name, tag)
|
||||
}
|
||||
|
||||
func dumpBox(box bmff.Box, depth int) {
|
||||
indent := strings.Repeat(" ", depth)
|
||||
fmt.Printf("%sBox: type %q, size %v\n", indent, box.Type(), box.Size())
|
||||
|
||||
box2, err := box.Parse()
|
||||
if err == bmff.ErrUnknownBox {
|
||||
slurp, err := ioutil.ReadAll(box.Body())
|
||||
if err != nil {
|
||||
log.Fatalf("%sreading body: %v", indent, err)
|
||||
}
|
||||
if len(slurp) < 5000 {
|
||||
fmt.Printf("%s- contents: %q\n", indent, slurp)
|
||||
} else {
|
||||
fmt.Printf("%s- contents: (... %d bytes, starting with %q ...)\n", indent, len(slurp), slurp[:100])
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
slurp, _ := ioutil.ReadAll(box.Body())
|
||||
log.Fatalf("Parse box type %q: %v; slurp: %q", box.Type(), err, slurp)
|
||||
}
|
||||
|
||||
switch v := box2.(type) {
|
||||
case *bmff.FileTypeBox, *bmff.HandlerBox, *bmff.PrimaryItemBox:
|
||||
fmt.Printf("%s- %T: %+v\n", indent, v, v)
|
||||
case *bmff.MetaBox:
|
||||
fmt.Printf("%s- %T, %d children:\n", indent, v, len(v.Children))
|
||||
for _, child := range v.Children {
|
||||
dumpBox(child, depth+1)
|
||||
}
|
||||
case *bmff.ItemInfoBox:
|
||||
//slurp, _ := ioutil.ReadAll(box.Body())
|
||||
//fmt.Printf("%s- %T raw: %q\n", indent, v, slurp)
|
||||
fmt.Printf("%s- %T, %d children (%d in slice):\n", indent, v, v.Count, len(v.ItemInfos))
|
||||
for _, child := range v.ItemInfos {
|
||||
dumpBox(child, depth+1)
|
||||
}
|
||||
case *bmff.ItemInfoEntry:
|
||||
fmt.Printf("%s- %T, %+v\n", indent, v, v)
|
||||
if v.ItemType == "Exif" {
|
||||
exifItemID = v.ItemID
|
||||
}
|
||||
case *bmff.ItemPropertiesBox:
|
||||
fmt.Printf("%s- %T\n", indent, v)
|
||||
if v.PropertyContainer != nil {
|
||||
dumpBox(v.PropertyContainer, depth+1)
|
||||
}
|
||||
for _, child := range v.Associations {
|
||||
dumpBox(child, depth+1)
|
||||
}
|
||||
case *bmff.ItemPropertyAssociation:
|
||||
fmt.Printf("%s- %T: %d declared entries, %d parsed:\n", indent, v, v.EntryCount, len(v.Entries))
|
||||
for _, ai := range v.Entries {
|
||||
fmt.Printf("%s for Item ID %d, %d associations declared, %d parsed:\n", indent, ai.ItemID, ai.AssociationsCount, len(ai.Associations))
|
||||
for _, ass := range ai.Associations {
|
||||
fmt.Printf("%s index: %d, essential: %v\n", indent, ass.Index, ass.Essential)
|
||||
}
|
||||
}
|
||||
case *bmff.DataInformationBox:
|
||||
fmt.Printf("%s- %T\n", indent, v)
|
||||
for _, child := range v.Children {
|
||||
dumpBox(child, depth+1)
|
||||
}
|
||||
case *bmff.DataReferenceBox:
|
||||
fmt.Printf("%s- %T\n", indent, v)
|
||||
for _, child := range v.Children {
|
||||
dumpBox(child, depth+1)
|
||||
}
|
||||
case *bmff.ItemPropertyContainerBox:
|
||||
fmt.Printf("%s- %T\n", indent, v)
|
||||
for _, child := range v.Properties {
|
||||
dumpBox(child, depth+1)
|
||||
}
|
||||
case *bmff.ItemLocationBox:
|
||||
fmt.Printf("%s- %T: %d items declared, %d parsed:\n", indent, v, v.ItemCount, len(v.Items))
|
||||
for _, lbe := range v.Items {
|
||||
fmt.Printf("%s %+v\n", indent, lbe)
|
||||
if exifItemID != 0 && lbe.ItemID == exifItemID {
|
||||
exifLoc = lbe
|
||||
}
|
||||
}
|
||||
|
||||
case *bmff.ImageSpatialExtentsProperty:
|
||||
fmt.Printf("%s- %T dimensions: %d x %d\n", indent, v, v.ImageWidth, v.ImageHeight)
|
||||
default:
|
||||
fmt.Printf("%s- gotype: %T\n", indent, box2)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,292 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package heif reads HEIF containers, as found in Apple HEIC/HEVC images.
|
||||
// This package does not decode images; it only reads the metadata.
|
||||
//
|
||||
// This package is a work in progress and makes no API compatibility
|
||||
// promises.
|
||||
package heif
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
|
||||
"go4.org/media/heif/bmff"
|
||||
)
|
||||
|
||||
// File represents a HEIF file.
|
||||
//
|
||||
// Methods on File should not be called concurrently.
|
||||
type File struct {
|
||||
ra io.ReaderAt
|
||||
primary *Item
|
||||
|
||||
// Populated lazily, by getMeta:
|
||||
metaErr error
|
||||
meta *BoxMeta
|
||||
}
|
||||
|
||||
// BoxMeta contains the low-level BMFF metadata boxes.
|
||||
type BoxMeta struct {
|
||||
FileType *bmff.FileTypeBox
|
||||
Handler *bmff.HandlerBox
|
||||
PrimaryItem *bmff.PrimaryItemBox
|
||||
ItemInfo *bmff.ItemInfoBox
|
||||
Properties *bmff.ItemPropertiesBox
|
||||
ItemLocation *bmff.ItemLocationBox
|
||||
}
|
||||
|
||||
// EXIFItemID returns the item ID of the EXIF part, or 0 if not found.
|
||||
func (m *BoxMeta) EXIFItemID() uint32 {
|
||||
if m.ItemInfo == nil {
|
||||
return 0
|
||||
}
|
||||
for _, ife := range m.ItemInfo.ItemInfos {
|
||||
if ife.ItemType == "Exif" {
|
||||
return uint32(ife.ItemID)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Item represents an item in a HEIF file.
|
||||
type Item struct {
|
||||
f *File
|
||||
|
||||
ID uint32
|
||||
Info *bmff.ItemInfoEntry
|
||||
Location *bmff.ItemLocationBoxEntry // location in file
|
||||
Properties []bmff.Box
|
||||
}
|
||||
|
||||
// SpatialExtents returns the item's spatial extents property values, if present,
|
||||
// not correcting from any camera rotation metadata.
|
||||
func (it *Item) SpatialExtents() (width, height int, ok bool) {
|
||||
for _, p := range it.Properties {
|
||||
if p, ok := p.(*bmff.ImageSpatialExtentsProperty); ok {
|
||||
return int(p.ImageWidth), int(p.ImageHeight), true
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Rotations returns the number of 90 degree rotations counter-clockwise that this
|
||||
// image should be rendered at, in the range [0,3].
|
||||
func (it *Item) Rotations() int {
|
||||
for _, p := range it.Properties {
|
||||
if p, ok := p.(*bmff.ImageRotation); ok {
|
||||
return int(p.Angle)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// VisualDimensions returns the item's width and height after correcting
|
||||
// for any rotations.
|
||||
func (it *Item) VisualDimensions() (width, height int, ok bool) {
|
||||
width, height, ok = it.SpatialExtents()
|
||||
for i := 0; i < it.Rotations(); i++ {
|
||||
width, height = height, width
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: add HEIF imir (mirroring) accessor, like Image.SpatialExtents.
|
||||
|
||||
// Open returns a handle to access a HEIF file.
|
||||
func Open(f io.ReaderAt) *File {
|
||||
return &File{ra: f}
|
||||
}
|
||||
|
||||
// ErrNoEXIF is returned by File.EXIF when a file does not contain an EXIF item.
|
||||
var ErrNoEXIF = errors.New("heif: no EXIF found")
|
||||
|
||||
// ErrUnknownItem is returned by File.ItemByID for unknown items.
|
||||
var ErrUnknownItem = errors.New("heif: unknown item")
|
||||
|
||||
// EXIF returns the raw EXIF data from the file.
|
||||
// The error is ErrNoEXIF if the file did not contain EXIF.
|
||||
//
|
||||
// The raw EXIF data can be parsed by the
|
||||
// github.com/rwcarlsen/goexif/exif package's Decode function.
|
||||
func (f *File) EXIF() ([]byte, error) {
|
||||
meta, err := f.getMeta()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exifID := meta.EXIFItemID()
|
||||
if exifID == 0 {
|
||||
return nil, ErrNoEXIF
|
||||
}
|
||||
it, err := f.ItemByID(exifID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if it.Location == nil {
|
||||
return nil, errors.New("heif: file said it contained EXIF, but didn't say where")
|
||||
}
|
||||
if n := len(it.Location.Extents); n != 1 {
|
||||
return nil, fmt.Errorf("heif: expected 1 EXIF section, saw %d", n)
|
||||
}
|
||||
offLen := it.Location.Extents[0]
|
||||
const maxSize = 20 << 10 // 20MB of EXIF seems excessive; cap it for sanity
|
||||
if offLen.Length > maxSize {
|
||||
return nil, fmt.Errorf("heif: declared EXIF size %d exceeds threshold of %d bytes", offLen.Length, maxSize)
|
||||
}
|
||||
buf := make([]byte, offLen.Length-4)
|
||||
n, err := f.ra.ReadAt(buf, int64(offLen.Offset)+4) // TODO: why 4? did I miss something?
|
||||
if err != nil {
|
||||
log.Printf("Read %d bytes + %v: %q", n, err, buf)
|
||||
return nil, err
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func (f *File) setMetaErr(err error) error {
|
||||
if f.metaErr != nil {
|
||||
f.metaErr = err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *File) getMeta() (*BoxMeta, error) {
|
||||
if f.metaErr != nil {
|
||||
return nil, f.metaErr
|
||||
}
|
||||
if f.meta != nil {
|
||||
return f.meta, nil
|
||||
}
|
||||
const assumedMaxSize = 5 << 40 // arbitrary
|
||||
sr := io.NewSectionReader(f.ra, 0, assumedMaxSize)
|
||||
bmr := bmff.NewReader(sr)
|
||||
|
||||
meta := &BoxMeta{}
|
||||
|
||||
pbox, err := bmr.ReadAndParseBox(bmff.TypeFtyp)
|
||||
if err != nil {
|
||||
return nil, f.setMetaErr(err)
|
||||
}
|
||||
meta.FileType = pbox.(*bmff.FileTypeBox)
|
||||
|
||||
pbox, err = bmr.ReadAndParseBox(bmff.TypeMeta)
|
||||
if err != nil {
|
||||
return nil, f.setMetaErr(err)
|
||||
}
|
||||
metabox := pbox.(*bmff.MetaBox)
|
||||
|
||||
for _, box := range metabox.Children {
|
||||
boxp, err := box.Parse()
|
||||
if err == bmff.ErrUnknownBox {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, f.setMetaErr(err)
|
||||
}
|
||||
switch v := boxp.(type) {
|
||||
case *bmff.HandlerBox:
|
||||
meta.Handler = v
|
||||
case *bmff.PrimaryItemBox:
|
||||
meta.PrimaryItem = v
|
||||
case *bmff.ItemInfoBox:
|
||||
meta.ItemInfo = v
|
||||
case *bmff.ItemPropertiesBox:
|
||||
meta.Properties = v
|
||||
case *bmff.ItemLocationBox:
|
||||
meta.ItemLocation = v
|
||||
}
|
||||
}
|
||||
|
||||
f.meta = meta
|
||||
return f.meta, nil
|
||||
}
|
||||
|
||||
// PrimaryItem returns the HEIF file's primary item.
|
||||
func (f *File) PrimaryItem() (*Item, error) {
|
||||
meta, err := f.getMeta()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if meta.PrimaryItem == nil {
|
||||
return nil, errors.New("heif: HEIF file lacks primary item box")
|
||||
}
|
||||
return f.ItemByID(uint32(meta.PrimaryItem.ItemID))
|
||||
}
|
||||
|
||||
// ItemByID by returns the file's Item of a given ID.
|
||||
// If the ID is known, the returned error is ErrUnknownItem.
|
||||
func (f *File) ItemByID(id uint32) (*Item, error) {
|
||||
meta, err := f.getMeta()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
it := &Item{
|
||||
f: f,
|
||||
ID: id,
|
||||
}
|
||||
if meta.ItemLocation != nil {
|
||||
for _, ilbe := range meta.ItemLocation.Items {
|
||||
if uint32(ilbe.ItemID) == id {
|
||||
shallowCopy := ilbe
|
||||
it.Location = &shallowCopy
|
||||
}
|
||||
}
|
||||
}
|
||||
if meta.ItemInfo != nil {
|
||||
for _, iie := range meta.ItemInfo.ItemInfos {
|
||||
if uint32(iie.ItemID) == id {
|
||||
it.Info = iie
|
||||
}
|
||||
}
|
||||
}
|
||||
if it.Info == nil {
|
||||
return nil, ErrUnknownItem
|
||||
}
|
||||
if meta.Properties != nil {
|
||||
allProps := meta.Properties.PropertyContainer.Properties
|
||||
for _, ipa := range meta.Properties.Associations {
|
||||
// TODO: I've never seen a file with more than
|
||||
// top-level ItemPropertyAssociation box, but
|
||||
// apparently they can exist with different
|
||||
// versions/flags. For now we just merge them
|
||||
// all together, but that's not really right.
|
||||
// So for now, just bail once a previous loop
|
||||
// found anything.
|
||||
if len(it.Properties) > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, ipai := range ipa.Entries {
|
||||
if ipai.ItemID != id {
|
||||
continue
|
||||
}
|
||||
for _, ass := range ipai.Associations {
|
||||
if ass.Index != 0 && int(ass.Index) <= len(allProps) {
|
||||
box := allProps[ass.Index-1]
|
||||
boxp, err := box.Parse()
|
||||
if err == nil {
|
||||
box = boxp
|
||||
}
|
||||
it.Properties = append(it.Properties, box)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return it, nil
|
||||
}
|
||||
@ -1,113 +0,0 @@
|
||||
package heif
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/rwcarlsen/goexif/exif"
|
||||
"github.com/rwcarlsen/goexif/tiff"
|
||||
)
|
||||
|
||||
func TestAll(t *testing.T) {
|
||||
f, err := os.Open("testdata/park.heic")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
h := Open(f)
|
||||
|
||||
// meta
|
||||
_, err = h.getMeta()
|
||||
if err != nil {
|
||||
t.Fatalf("getMeta: %v", err)
|
||||
}
|
||||
|
||||
it, err := h.PrimaryItem()
|
||||
if err != nil {
|
||||
t.Fatalf("PrimaryItem: %v", err)
|
||||
}
|
||||
if want := uint32(49); it.ID != want {
|
||||
t.Errorf("PrimaryIem ID = %v; want %v", it.ID, want)
|
||||
}
|
||||
if it.Location == nil {
|
||||
t.Errorf("Item.Location is nil")
|
||||
}
|
||||
if it.Info == nil {
|
||||
t.Errorf("Item.Info is nil")
|
||||
}
|
||||
if len(it.Properties) == 0 {
|
||||
t.Errorf("Item.Properties is empty")
|
||||
}
|
||||
for _, prop := range it.Properties {
|
||||
t.Logf(" property: %q, %#v", prop.Type(), prop)
|
||||
}
|
||||
if w, h, ok := it.SpatialExtents(); !ok || w == 0 || h == 0 {
|
||||
t.Errorf("no spatial extents found")
|
||||
} else {
|
||||
t.Logf("dimensions: %v x %v", w, h)
|
||||
}
|
||||
|
||||
// exif
|
||||
exbuf, err := h.EXIF()
|
||||
if err != nil {
|
||||
t.Errorf("EXIF: %v", err)
|
||||
} else {
|
||||
const magic = "Exif\x00\x00"
|
||||
if !bytes.HasPrefix(exbuf, []byte(magic)) {
|
||||
t.Errorf("Exif buffer doesn't start with %q: got %q", magic, exbuf)
|
||||
}
|
||||
x, err := exif.Decode(bytes.NewReader(exbuf))
|
||||
if err != nil {
|
||||
t.Fatalf("EXIF decode: %v", err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
if err := x.Walk(walkFunc(func(name exif.FieldName, tag *tiff.Tag) error {
|
||||
got[fmt.Sprint(name)] = fmt.Sprint(tag)
|
||||
return nil
|
||||
})); err != nil {
|
||||
t.Fatalf("EXIF walk: %v", err)
|
||||
}
|
||||
if g, w := len(got), 56; g < w {
|
||||
t.Errorf("saw %v EXIF tags; want at least %v", g, w)
|
||||
}
|
||||
if g, w := got["GPSLongitude"], `["122/1","21/1","3776/100"]`; g != w {
|
||||
t.Errorf("GPSLongitude = %#q; want %#q", g, w)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestRotations(t *testing.T) {
|
||||
f, err := os.Open("testdata/rotate.heic")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
h := Open(f)
|
||||
it, err := h.PrimaryItem()
|
||||
if err != nil {
|
||||
t.Fatalf("PrimaryItem: %v", err)
|
||||
}
|
||||
if r := it.Rotations(); r != 3 {
|
||||
t.Errorf("Rotations = %v; want %v", r, 3)
|
||||
}
|
||||
sw, sh, ok := it.SpatialExtents()
|
||||
if !ok {
|
||||
t.Fatalf("expected spatial extents")
|
||||
}
|
||||
vw, vh, ok := it.VisualDimensions()
|
||||
if !ok {
|
||||
t.Fatalf("expected visual dimensions")
|
||||
}
|
||||
if vw != sh || vh != sw {
|
||||
t.Errorf("visual dimensions = %v, %v; want %v, %v", vw, vh, sh, sw)
|
||||
}
|
||||
}
|
||||
|
||||
type walkFunc func(exif.FieldName, *tiff.Tag) error
|
||||
|
||||
func (f walkFunc) Walk(name exif.FieldName, tag *tiff.Tag) error {
|
||||
return f(name, tag)
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
@ -1,137 +0,0 @@
|
||||
/*
|
||||
Copyright 2012 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package throttle provides a net.Listener that returns
|
||||
// artificially-delayed connections for testing real-world
|
||||
// connectivity.
|
||||
package throttle // import "go4.org/net/throttle"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const unitSize = 1400 // read/write chunk size. ~MTU size.
|
||||
|
||||
type Rate struct {
|
||||
KBps int // or 0, to not rate-limit bandwidth
|
||||
Latency time.Duration
|
||||
}
|
||||
|
||||
// byteTime returns the time required for n bytes.
|
||||
func (r Rate) byteTime(n int) time.Duration {
|
||||
if r.KBps == 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(float64(n)/1024/float64(r.KBps)) * time.Second
|
||||
}
|
||||
|
||||
type Listener struct {
|
||||
net.Listener
|
||||
Down Rate // server Writes to Client
|
||||
Up Rate // server Reads from client
|
||||
}
|
||||
|
||||
func (ln *Listener) Accept() (net.Conn, error) {
|
||||
c, err := ln.Listener.Accept()
|
||||
time.Sleep(ln.Up.Latency)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tc := &conn{Conn: c, Down: ln.Down, Up: ln.Up}
|
||||
tc.start()
|
||||
return tc, nil
|
||||
}
|
||||
|
||||
type nErr struct {
|
||||
n int
|
||||
err error
|
||||
}
|
||||
|
||||
type writeReq struct {
|
||||
writeAt time.Time
|
||||
p []byte
|
||||
resc chan nErr
|
||||
}
|
||||
|
||||
type conn struct {
|
||||
net.Conn
|
||||
Down Rate // for reads
|
||||
Up Rate // for writes
|
||||
|
||||
wchan chan writeReq
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (c *conn) start() {
|
||||
c.wchan = make(chan writeReq, 1024)
|
||||
go c.writeLoop()
|
||||
}
|
||||
|
||||
func (c *conn) writeLoop() {
|
||||
for req := range c.wchan {
|
||||
time.Sleep(req.writeAt.Sub(time.Now()))
|
||||
var res nErr
|
||||
for len(req.p) > 0 && res.err == nil {
|
||||
writep := req.p
|
||||
if len(writep) > unitSize {
|
||||
writep = writep[:unitSize]
|
||||
}
|
||||
n, err := c.Conn.Write(writep)
|
||||
time.Sleep(c.Up.byteTime(len(writep)))
|
||||
res.n += n
|
||||
res.err = err
|
||||
req.p = req.p[n:]
|
||||
}
|
||||
req.resc <- res
|
||||
}
|
||||
}
|
||||
|
||||
func (c *conn) Close() error {
|
||||
c.closeOnce.Do(func() {
|
||||
err := c.Conn.Close()
|
||||
close(c.wchan)
|
||||
c.closeErr = err
|
||||
})
|
||||
return c.closeErr
|
||||
}
|
||||
|
||||
func (c *conn) Write(p []byte) (n int, err error) {
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
n = 0
|
||||
err = fmt.Errorf("%v", err)
|
||||
return
|
||||
}
|
||||
}()
|
||||
resc := make(chan nErr, 1)
|
||||
c.wchan <- writeReq{time.Now().Add(c.Up.Latency), p, resc}
|
||||
res := <-resc
|
||||
return res.n, res.err
|
||||
}
|
||||
|
||||
func (c *conn) Read(p []byte) (n int, err error) {
|
||||
const max = 1024
|
||||
if len(p) > max {
|
||||
p = p[:max]
|
||||
}
|
||||
n, err = c.Conn.Read(p)
|
||||
time.Sleep(c.Down.byteTime(n))
|
||||
return
|
||||
}
|
||||
@ -1,121 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package oauthutil contains OAuth 2 related utilities.
|
||||
package oauthutil // import "go4.org/oauthutil"
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go4.org/wkfs"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
// TitleBarRedirectURL is the OAuth2 redirect URL to use when the authorization
|
||||
// code should be returned in the title bar of the browser, with the page text
|
||||
// prompting the user to copy the code and paste it in the application.
|
||||
const TitleBarRedirectURL = "urn:ietf:wg:oauth:2.0:oob"
|
||||
|
||||
// ErrNoAuthCode is returned when Token() has not found any valid cached token
|
||||
// and TokenSource does not have an AuthCode for getting a new token.
|
||||
var ErrNoAuthCode = errors.New("oauthutil: unspecified TokenSource.AuthCode")
|
||||
|
||||
// TokenSource is an implementation of oauth2.TokenSource. It uses CacheFile to store and
|
||||
// reuse the the acquired token, and AuthCode to provide the authorization code that will be
|
||||
// exchanged for a token otherwise.
|
||||
type TokenSource struct {
|
||||
Config *oauth2.Config
|
||||
|
||||
// CacheFile is where the token will be stored JSON-encoded. Any call to Token
|
||||
// first tries to read a valid token from CacheFile.
|
||||
CacheFile string
|
||||
|
||||
// AuthCode provides the authorization code that Token will exchange for a token.
|
||||
// It usually is a way to prompt the user for the code. If CacheFile does not provide
|
||||
// a token and AuthCode is nil, Token returns ErrNoAuthCode.
|
||||
AuthCode func() string
|
||||
}
|
||||
|
||||
var errExpiredToken = errors.New("expired token")
|
||||
|
||||
// cachedToken returns the token saved in cacheFile. It specifically returns
|
||||
// errTokenExpired if the token is expired.
|
||||
func cachedToken(cacheFile string) (*oauth2.Token, error) {
|
||||
tok := new(oauth2.Token)
|
||||
tokenData, err := wkfs.ReadFile(cacheFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = json.Unmarshal(tokenData, tok); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !tok.Valid() {
|
||||
if tok != nil && time.Now().After(tok.Expiry) {
|
||||
return nil, errExpiredToken
|
||||
}
|
||||
return nil, errors.New("invalid token")
|
||||
}
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
// Token first tries to find a valid token in CacheFile, and otherwise uses
|
||||
// Config and AuthCode to fetch a new token. This new token is saved in CacheFile
|
||||
// (if not blank). If CacheFile did not provide a token and AuthCode is nil,
|
||||
// ErrNoAuthCode is returned.
|
||||
func (src TokenSource) Token() (*oauth2.Token, error) {
|
||||
var tok *oauth2.Token
|
||||
var err error
|
||||
if src.CacheFile != "" {
|
||||
tok, err = cachedToken(src.CacheFile)
|
||||
if err == nil {
|
||||
return tok, nil
|
||||
}
|
||||
if err != errExpiredToken {
|
||||
fmt.Printf("Error getting token from %s: %v\n", src.CacheFile, err)
|
||||
}
|
||||
}
|
||||
if src.AuthCode == nil {
|
||||
return nil, ErrNoAuthCode
|
||||
}
|
||||
tok, err = src.Config.Exchange(oauth2.NoContext, src.AuthCode())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not exchange auth code for a token: %v", err)
|
||||
}
|
||||
if src.CacheFile == "" {
|
||||
return tok, nil
|
||||
}
|
||||
tokenData, err := json.Marshal(&tok)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not encode token as json: %v", err)
|
||||
}
|
||||
if err := wkfs.WriteFile(src.CacheFile, tokenData, 0600); err != nil {
|
||||
return nil, fmt.Errorf("could not cache token in %v: %v", src.CacheFile, err)
|
||||
}
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
// NewRefreshTokenSource returns a token source that obtains its initial token
|
||||
// based on the provided config and the refresh token.
|
||||
func NewRefreshTokenSource(config *oauth2.Config, refreshToken string) oauth2.TokenSource {
|
||||
var noInitialToken *oauth2.Token = nil
|
||||
return oauth2.ReuseTokenSource(noInitialToken, config.TokenSource(
|
||||
oauth2.NoContext, // TODO: maybe accept a context later.
|
||||
&oauth2.Token{RefreshToken: refreshToken},
|
||||
))
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
// Copyright 2015 The go4 Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build plan9
|
||||
|
||||
package osutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func executable() (string, error) {
|
||||
fn := fmt.Sprintf("/proc/%d/text", os.Getpid())
|
||||
f, err := os.Open(fn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
p, err := syscall.Fd2path(int(f.Fd()))
|
||||
return filepath.Clean(p), err
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
// Copyright 2015 The go4 Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build linux netbsd openbsd dragonfly nacl
|
||||
|
||||
package osutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func executable() (string, error) {
|
||||
var procfn string
|
||||
switch runtime.GOOS {
|
||||
default:
|
||||
return "", errors.New("Executable not implemented for " + runtime.GOOS)
|
||||
case "linux":
|
||||
procfn = "/proc/self/exe"
|
||||
case "netbsd":
|
||||
procfn = "/proc/curproc/exe"
|
||||
case "openbsd":
|
||||
procfn = "/proc/curproc/file"
|
||||
case "dragonfly":
|
||||
procfn = "/proc/curproc/file"
|
||||
}
|
||||
p, err := os.Readlink(procfn)
|
||||
return filepath.Clean(p), err
|
||||
}
|
||||
@ -1,71 +0,0 @@
|
||||
// Copyright 2015 The go4 Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build amd64,solaris
|
||||
|
||||
package osutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
//go:cgo_import_dynamic libc_getexecname getexecname "libc.so"
|
||||
//go:linkname libc_getexecname libc_getexecname
|
||||
|
||||
var libc_getexecname uintptr
|
||||
|
||||
func getexecname() (path unsafe.Pointer, err error) {
|
||||
r0, _, e1 := syscall.Syscall6(uintptr(unsafe.Pointer(&libc_getexecname)), 0, 0, 0, 0, 0, 0)
|
||||
path = unsafe.Pointer(r0)
|
||||
if e1 != 0 {
|
||||
err = syscall.Errno(e1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func syscallGetexecname() (path string, err error) {
|
||||
ptr, err := getexecname()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
bytes := (*[1 << 29]byte)(ptr)[:]
|
||||
for i, b := range bytes {
|
||||
if b == 0 {
|
||||
return string(bytes[:i]), nil
|
||||
}
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
var initCwd, initCwdErr = os.Getwd()
|
||||
|
||||
func executable() (string, error) {
|
||||
path, err := syscallGetexecname()
|
||||
if err != nil {
|
||||
return path, err
|
||||
}
|
||||
if len(path) > 0 && path[0] != '/' {
|
||||
if initCwdErr != nil {
|
||||
return path, initCwdErr
|
||||
}
|
||||
if len(path) > 2 && path[0:2] == "./" {
|
||||
// skip "./"
|
||||
path = path[2:]
|
||||
}
|
||||
return initCwd + "/" + path, nil
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
// Copyright 2015 The go4 Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build freebsd darwin
|
||||
|
||||
package osutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var cacheWD, cacheWDErr = os.Getwd()
|
||||
|
||||
func executable() (string, error) {
|
||||
var mib [4]int32
|
||||
switch runtime.GOOS {
|
||||
case "freebsd":
|
||||
mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
|
||||
case "darwin":
|
||||
mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
|
||||
}
|
||||
|
||||
n := uintptr(0)
|
||||
// get length
|
||||
_, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
|
||||
if err != 0 {
|
||||
return "", err
|
||||
}
|
||||
if n == 0 { // shouldn't happen
|
||||
return "", nil
|
||||
}
|
||||
buf := make([]byte, n)
|
||||
_, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
|
||||
if err != 0 {
|
||||
return "", err
|
||||
}
|
||||
if n == 0 { // shouldn't happen
|
||||
return "", nil
|
||||
}
|
||||
p := string(buf[:n-1])
|
||||
if !filepath.IsAbs(p) {
|
||||
if cacheWDErr != nil {
|
||||
return p, cacheWDErr
|
||||
}
|
||||
p = filepath.Join(cacheWD, filepath.Clean(p))
|
||||
}
|
||||
return filepath.EvalSymlinks(p)
|
||||
}
|
||||
@ -1,94 +0,0 @@
|
||||
// Copyright 2015 The go4 Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package osutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
osexec "os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const executable_EnvVar = "OSTEST_OUTPUT_EXECPATH"
|
||||
|
||||
func TestExecutable(t *testing.T) {
|
||||
if runtime.GOOS == "nacl" {
|
||||
t.Skip()
|
||||
}
|
||||
ep, err := Executable()
|
||||
if err != nil {
|
||||
switch goos := runtime.GOOS; goos {
|
||||
case "openbsd": // procfs is not mounted by default
|
||||
t.Skipf("Executable failed on %s: %v, expected", goos, err)
|
||||
}
|
||||
t.Fatalf("Executable failed: %v", err)
|
||||
}
|
||||
// we want fn to be of the form "dir/prog"
|
||||
dir := filepath.Dir(filepath.Dir(ep))
|
||||
fn, err := filepath.Rel(dir, ep)
|
||||
if err != nil {
|
||||
t.Fatalf("filepath.Rel: %v", err)
|
||||
}
|
||||
cmd := &osexec.Cmd{}
|
||||
// make child start with a relative program path
|
||||
cmd.Dir = dir
|
||||
cmd.Path = fn
|
||||
// forge argv[0] for child, so that we can verify we could correctly
|
||||
// get real path of the executable without influenced by argv[0].
|
||||
cmd.Args = []string{"-", "-test.run=XXXX"}
|
||||
cmd.Env = []string{fmt.Sprintf("%s=1", executable_EnvVar)}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("exec(self) failed: %v", err)
|
||||
}
|
||||
outs := string(out)
|
||||
if !filepath.IsAbs(outs) {
|
||||
t.Fatalf("Child returned %q, want an absolute path", out)
|
||||
}
|
||||
if !sameFile(outs, ep) {
|
||||
t.Fatalf("Child returned %q, not the same file as %q", out, ep)
|
||||
}
|
||||
}
|
||||
|
||||
func sameFile(fn1, fn2 string) bool {
|
||||
fi1, err := os.Stat(fn1)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
fi2, err := os.Stat(fn2)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return os.SameFile(fi1, fi2)
|
||||
}
|
||||
|
||||
func init() {
|
||||
if e := os.Getenv(executable_EnvVar); e != "" {
|
||||
// first chdir to another path
|
||||
dir := "/"
|
||||
if runtime.GOOS == "windows" {
|
||||
dir = filepath.VolumeName(".")
|
||||
}
|
||||
os.Chdir(dir)
|
||||
if ep, err := Executable(); err != nil {
|
||||
fmt.Fprint(os.Stderr, "ERROR: ", err)
|
||||
} else {
|
||||
fmt.Fprint(os.Stderr, ep)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package osutil
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var (
|
||||
modkernel32 = syscall.MustLoadDLL("kernel32.dll")
|
||||
procGetModuleFileNameW = modkernel32.MustFindProc("GetModuleFileNameW")
|
||||
)
|
||||
|
||||
func getModuleFileName(handle syscall.Handle) (string, error) {
|
||||
n := uint32(1024)
|
||||
var buf []uint16
|
||||
for {
|
||||
buf = make([]uint16, n)
|
||||
r, err := syscallGetModuleFileName(handle, &buf[0], n)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if r < n {
|
||||
break
|
||||
}
|
||||
// r == n means n not big enough
|
||||
n += 1024
|
||||
}
|
||||
return syscall.UTF16ToString(buf), nil
|
||||
}
|
||||
|
||||
func executable() (string, error) {
|
||||
p, err := getModuleFileName(0)
|
||||
return filepath.Clean(p), err
|
||||
}
|
||||
|
||||
func syscallGetModuleFileName(module syscall.Handle, fn *uint16, len uint32) (n uint32, err error) {
|
||||
r0, _, e1 := syscall.Syscall(procGetModuleFileNameW.Addr(), 3, uintptr(module), uintptr(unsafe.Pointer(fn)), uintptr(len))
|
||||
n = uint32(r0)
|
||||
if n == 0 {
|
||||
if e1 != 0 {
|
||||
err = error(e1)
|
||||
} else {
|
||||
err = syscall.EINVAL
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package osutil contains os level functions.
|
||||
package osutil // import "go4.org/osutil"
|
||||
|
||||
// capture executable on package init to work around various os issues if
|
||||
// captured after executable has been renamed.
|
||||
var execPath, execError = executable()
|
||||
|
||||
// Executable returns the path name for the executable that starts the
|
||||
// current process. The result is the path that was used to start the
|
||||
// current process, but there is no guarantee that the path is still
|
||||
// pointing to the correct executable.
|
||||
//
|
||||
// OpenBSD is currently unsupported.
|
||||
func Executable() (string, error) {
|
||||
return execPath, execError
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import "io"
|
||||
|
||||
// NewBufferingReaderAt returns an io.ReaderAt that reads from r as
|
||||
// necessary and keeps a copy of all data read in memory.
|
||||
func NewBufferingReaderAt(r io.Reader) io.ReaderAt {
|
||||
return &bufReaderAt{r: r}
|
||||
}
|
||||
|
||||
type bufReaderAt struct {
|
||||
r io.Reader
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func (br *bufReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
endOff := off + int64(len(p))
|
||||
need := endOff - int64(len(br.buf))
|
||||
if need > 0 {
|
||||
buf := make([]byte, need)
|
||||
var rn int
|
||||
rn, err = io.ReadFull(br.r, buf)
|
||||
br.buf = append(br.buf, buf[:rn]...)
|
||||
}
|
||||
if int64(len(br.buf)) >= off {
|
||||
n = copy(p, br.buf[off:])
|
||||
}
|
||||
if n == len(p) {
|
||||
err = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
@ -1,70 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import "testing"
|
||||
|
||||
type trackingReader struct {
|
||||
off int
|
||||
reads int
|
||||
readBytes int
|
||||
}
|
||||
|
||||
func (t *trackingReader) Read(p []byte) (n int, err error) {
|
||||
t.reads++
|
||||
t.readBytes += len(p)
|
||||
for len(p) > 0 {
|
||||
p[0] = '0' + byte(t.off%10)
|
||||
t.off++
|
||||
p = p[1:]
|
||||
n++
|
||||
}
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func TestBufferingReaderAt(t *testing.T) {
|
||||
tr := new(trackingReader)
|
||||
ra := NewBufferingReaderAt(tr)
|
||||
for i, tt := range []struct {
|
||||
off int64
|
||||
want string
|
||||
wantReads int
|
||||
wantReadBytes int
|
||||
}{
|
||||
{off: 0, want: "0123456789", wantReads: 1, wantReadBytes: 10},
|
||||
{off: 5, want: "56789", wantReads: 1, wantReadBytes: 10}, // already buffered
|
||||
{off: 6, want: "67890", wantReads: 2, wantReadBytes: 11}, // need 1 more byte
|
||||
{off: 0, want: "0123456789", wantReads: 2, wantReadBytes: 11}, // already buffered
|
||||
} {
|
||||
got := make([]byte, len(tt.want))
|
||||
n, err := ra.ReadAt(got, tt.off)
|
||||
if err != nil || n != len(tt.want) {
|
||||
t.Errorf("step %d: ReadAt = %v, %v; want %v, %v", i, n, err, len(tt.want), nil)
|
||||
continue
|
||||
}
|
||||
if string(got) != tt.want {
|
||||
t.Errorf("step %d: ReadAt read %q; want %q", i, got, tt.want)
|
||||
}
|
||||
if tr.reads != tt.wantReads {
|
||||
t.Errorf("step %d: num reads = %d; want %d", i, tr.reads, tt.wantReads)
|
||||
}
|
||||
if tr.readBytes != tt.wantReadBytes {
|
||||
t.Errorf("step %d: read bytes = %d; want %d", i, tr.reads, tt.wantReads)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
/*
|
||||
Copyright 2011 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import "io"
|
||||
|
||||
// CountingReader wraps a Reader, incrementing N by the number of
|
||||
// bytes read. No locking is performed.
|
||||
type CountingReader struct {
|
||||
Reader io.Reader
|
||||
N *int64
|
||||
}
|
||||
|
||||
func (cr CountingReader) Read(p []byte) (n int, err error) {
|
||||
n, err = cr.Reader.Read(p)
|
||||
*cr.N += int64(n)
|
||||
return
|
||||
}
|
||||
@ -1,70 +0,0 @@
|
||||
/*
|
||||
Copyright 2014 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// fakeSeeker can seek to the ends but any read not at the current
|
||||
// position will fail.
|
||||
type fakeSeeker struct {
|
||||
r io.Reader
|
||||
size int64
|
||||
|
||||
fakePos int64
|
||||
realPos int64
|
||||
}
|
||||
|
||||
// NewFakeSeeker returns a ReadSeeker that can pretend to Seek (based
|
||||
// on the provided total size of the reader's content), but any reads
|
||||
// will fail if the fake seek position doesn't match reality.
|
||||
func NewFakeSeeker(r io.Reader, size int64) io.ReadSeeker {
|
||||
return &fakeSeeker{r: r, size: size}
|
||||
}
|
||||
|
||||
func (fs *fakeSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
var newo int64
|
||||
switch whence {
|
||||
default:
|
||||
return 0, errors.New("invalid whence")
|
||||
case os.SEEK_SET:
|
||||
newo = offset
|
||||
case os.SEEK_CUR:
|
||||
newo = fs.fakePos + offset
|
||||
case os.SEEK_END:
|
||||
newo = fs.size + offset
|
||||
}
|
||||
if newo < 0 {
|
||||
return 0, errors.New("negative seek")
|
||||
}
|
||||
fs.fakePos = newo
|
||||
return newo, nil
|
||||
}
|
||||
|
||||
func (fs *fakeSeeker) Read(p []byte) (n int, err error) {
|
||||
if fs.fakePos != fs.realPos {
|
||||
return 0, fmt.Errorf("attempt to read from fake seek offset %d; real offset is %d", fs.fakePos, fs.realPos)
|
||||
}
|
||||
n, err = fs.r.Read(p)
|
||||
fs.fakePos += int64(n)
|
||||
fs.realPos += int64(n)
|
||||
return
|
||||
}
|
||||
@ -1,55 +0,0 @@
|
||||
/*
|
||||
Copyright 2014 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFakeSeeker(t *testing.T) {
|
||||
rs := NewFakeSeeker(strings.NewReader("foobar"), 6)
|
||||
if pos, err := rs.Seek(0, os.SEEK_END); err != nil || pos != 6 {
|
||||
t.Fatalf("SEEK_END = %d, %v; want 6, nil", pos, err)
|
||||
}
|
||||
if pos, err := rs.Seek(0, os.SEEK_CUR); err != nil || pos != 6 {
|
||||
t.Fatalf("SEEK_CUR = %d, %v; want 6, nil", pos, err)
|
||||
}
|
||||
if pos, err := rs.Seek(0, os.SEEK_SET); err != nil || pos != 0 {
|
||||
t.Fatalf("SEEK_SET = %d, %v; want 0, nil", pos, err)
|
||||
}
|
||||
|
||||
buf := make([]byte, 3)
|
||||
if n, err := rs.Read(buf); n != 3 || err != nil || string(buf) != "foo" {
|
||||
t.Fatalf("First read = %d, %v (buf = %q); want foo", n, err, buf)
|
||||
}
|
||||
if pos, err := rs.Seek(0, os.SEEK_CUR); err != nil || pos != 3 {
|
||||
t.Fatalf("Seek cur pos after first read = %d, %v; want 3, nil", pos, err)
|
||||
}
|
||||
if n, err := rs.Read(buf); n != 3 || err != nil || string(buf) != "bar" {
|
||||
t.Fatalf("Second read = %d, %v (buf = %q); want foo", n, err, buf)
|
||||
}
|
||||
|
||||
if pos, err := rs.Seek(1, os.SEEK_SET); err != nil || pos != 1 {
|
||||
t.Fatalf("SEEK_SET = %d, %v; want 1, nil", pos, err)
|
||||
}
|
||||
const msg = "attempt to read from fake seek offset"
|
||||
if _, err := rs.Read(buf); err == nil || !strings.Contains(err.Error(), msg) {
|
||||
t.Fatalf("bogus Read after seek = %v; want something containing %q", err, msg)
|
||||
}
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// NewMultiReaderAt is like io.MultiReader but produces a ReaderAt
|
||||
// (and Size), instead of just a reader.
|
||||
func NewMultiReaderAt(parts ...SizeReaderAt) SizeReaderAt {
|
||||
m := &multiRA{
|
||||
parts: make([]offsetAndSource, 0, len(parts)),
|
||||
}
|
||||
var off int64
|
||||
for _, p := range parts {
|
||||
m.parts = append(m.parts, offsetAndSource{off, p})
|
||||
off += p.Size()
|
||||
}
|
||||
m.size = off
|
||||
return m
|
||||
}
|
||||
|
||||
type offsetAndSource struct {
|
||||
off int64
|
||||
SizeReaderAt
|
||||
}
|
||||
|
||||
type multiRA struct {
|
||||
parts []offsetAndSource
|
||||
size int64
|
||||
}
|
||||
|
||||
func (m *multiRA) Size() int64 { return m.size }
|
||||
|
||||
func (m *multiRA) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
wantN := len(p)
|
||||
|
||||
// Skip past the requested offset.
|
||||
skipParts := sort.Search(len(m.parts), func(i int) bool {
|
||||
// This function returns whether parts[i] will
|
||||
// contribute any bytes to our output.
|
||||
part := m.parts[i]
|
||||
return part.off+part.Size() > off
|
||||
})
|
||||
parts := m.parts[skipParts:]
|
||||
|
||||
// How far to skip in the first part.
|
||||
needSkip := off
|
||||
if len(parts) > 0 {
|
||||
needSkip -= parts[0].off
|
||||
}
|
||||
|
||||
for len(parts) > 0 && len(p) > 0 {
|
||||
readP := p
|
||||
partSize := parts[0].Size()
|
||||
if int64(len(readP)) > partSize-needSkip {
|
||||
readP = readP[:partSize-needSkip]
|
||||
}
|
||||
pn, err0 := parts[0].ReadAt(readP, needSkip)
|
||||
if err0 != nil {
|
||||
return n, err0
|
||||
}
|
||||
n += pn
|
||||
p = p[pn:]
|
||||
if int64(pn)+needSkip == partSize {
|
||||
parts = parts[1:]
|
||||
}
|
||||
needSkip = 0
|
||||
}
|
||||
|
||||
if n != wantN {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Go4 Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMultiReaderAt(t *testing.T) {
|
||||
sra := NewMultiReaderAt(
|
||||
io.NewSectionReader(strings.NewReader("xaaax"), 1, 3),
|
||||
io.NewSectionReader(strings.NewReader("xxbbbbxx"), 2, 3),
|
||||
io.NewSectionReader(strings.NewReader("cccx"), 0, 3),
|
||||
)
|
||||
if sra.Size() != 9 {
|
||||
t.Fatalf("Size = %d; want 9", sra.Size())
|
||||
}
|
||||
const full = "aaabbbccc"
|
||||
for start := 0; start < len(full); start++ {
|
||||
for end := start; end < len(full); end++ {
|
||||
want := full[start:end]
|
||||
got, err := ioutil.ReadAll(io.NewSectionReader(sra, int64(start), int64(end-start)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != want {
|
||||
t.Errorf("for start=%d, end=%d: ReadAll = %q; want %q", start, end, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
/*
|
||||
Copyright 2012 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package readerutil provides and operates on io.Readers.
|
||||
package readerutil // import "go4.org/readerutil"
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Size tries to determine the length of r. If r is an io.Seeker, Size may seek
|
||||
// to guess the length.
|
||||
func Size(r io.Reader) (size int64, ok bool) {
|
||||
switch rt := r.(type) {
|
||||
case *bytes.Buffer:
|
||||
return int64(rt.Len()), true
|
||||
case *bytes.Reader:
|
||||
return int64(rt.Len()), true
|
||||
case *strings.Reader:
|
||||
return int64(rt.Len()), true
|
||||
case io.Seeker:
|
||||
pos, err := rt.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
end, err := rt.Seek(0, os.SEEK_END)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
size = end - pos
|
||||
pos1, err := rt.Seek(pos, os.SEEK_SET)
|
||||
if err != nil || pos1 != pos {
|
||||
msg := "failed to restore seek position"
|
||||
if err != nil {
|
||||
msg += ": " + err.Error()
|
||||
}
|
||||
panic(msg)
|
||||
}
|
||||
return size, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
@ -1,68 +0,0 @@
|
||||
/*
|
||||
Copyright 2012 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const text = "HelloWorld"
|
||||
|
||||
type testSrc struct {
|
||||
name string
|
||||
src io.Reader
|
||||
want int64
|
||||
}
|
||||
|
||||
func (tsrc *testSrc) run(t *testing.T) {
|
||||
n, ok := Size(tsrc.src)
|
||||
if !ok {
|
||||
t.Fatalf("failed to read size for %q", tsrc.name)
|
||||
}
|
||||
if n != tsrc.want {
|
||||
t.Fatalf("wanted %v, got %v", tsrc.want, n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesBuffer(t *testing.T) {
|
||||
buf := bytes.NewBuffer([]byte(text))
|
||||
tsrc := &testSrc{"buffer", buf, int64(len(text))}
|
||||
tsrc.run(t)
|
||||
}
|
||||
|
||||
func TestSeeker(t *testing.T) {
|
||||
f, err := ioutil.TempFile("", "camliTestReaderSize")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
size, err := f.Write([]byte(text))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pos, err := f.Seek(5, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tsrc := &testSrc{"seeker", f, int64(size) - pos}
|
||||
tsrc.run(t)
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package readerutil contains io.Reader types.
|
||||
package readerutil // import "go4.org/readerutil"
|
||||
|
||||
import (
|
||||
"expvar"
|
||||
"io"
|
||||
)
|
||||
|
||||
// A SizeReaderAt is a ReaderAt with a Size method.
|
||||
//
|
||||
// An io.SectionReader implements SizeReaderAt.
|
||||
type SizeReaderAt interface {
|
||||
Size() int64
|
||||
io.ReaderAt
|
||||
}
|
||||
|
||||
// A ReadSeekCloser can Read, Seek, and Close.
|
||||
type ReadSeekCloser interface {
|
||||
io.Reader
|
||||
io.Seeker
|
||||
io.Closer
|
||||
}
|
||||
|
||||
type ReaderAtCloser interface {
|
||||
io.ReaderAt
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// TODO(wathiede): make sure all the stat readers work with code that
|
||||
// type asserts ReadFrom/WriteTo.
|
||||
|
||||
type varStatReader struct {
|
||||
*expvar.Int
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
// NewReaderStats returns an io.Reader that will have the number of bytes
|
||||
// read from r added to v.
|
||||
func NewStatsReader(v *expvar.Int, r io.Reader) io.Reader {
|
||||
return &varStatReader{v, r}
|
||||
}
|
||||
|
||||
func (v *varStatReader) Read(p []byte) (int, error) {
|
||||
n, err := v.r.Read(p)
|
||||
v.Int.Add(int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
type varStatReadSeeker struct {
|
||||
*expvar.Int
|
||||
rs io.ReadSeeker
|
||||
}
|
||||
|
||||
// NewReaderStats returns an io.ReadSeeker that will have the number of bytes
|
||||
// read from rs added to v.
|
||||
func NewStatsReadSeeker(v *expvar.Int, rs io.ReadSeeker) io.ReadSeeker {
|
||||
return &varStatReadSeeker{v, rs}
|
||||
}
|
||||
|
||||
func (v *varStatReadSeeker) Read(p []byte) (int, error) {
|
||||
n, err := v.rs.Read(p)
|
||||
v.Int.Add(int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (v *varStatReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
return v.rs.Seek(offset, whence)
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Go4 Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package readerutil
|
||||
|
||||
import (
|
||||
"expvar"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ExampleNewStatsReader() {
|
||||
var (
|
||||
// r is the io.Reader we'd like to count read from.
|
||||
r = strings.NewReader("Hello world")
|
||||
v = expvar.NewInt("read-bytes")
|
||||
sw = NewStatsReader(v, r)
|
||||
)
|
||||
// Read from the wrapped io.Reader, StatReader will count the bytes.
|
||||
io.Copy(ioutil.Discard, sw)
|
||||
fmt.Printf("Read %s bytes\n", v.String())
|
||||
// Output: Read 11 bytes
|
||||
}
|
||||
@ -1,118 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// package singlereader provides Open and Close operations, reusing existing
|
||||
// file descriptors when possible.
|
||||
package singlereader // import "go4.org/readerutil/singlereader"
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"go4.org/readerutil"
|
||||
"go4.org/syncutil/singleflight"
|
||||
"go4.org/wkfs"
|
||||
)
|
||||
|
||||
var (
|
||||
openerGroup singleflight.Group
|
||||
|
||||
openFileMu sync.Mutex // guards openFiles
|
||||
openFiles = make(map[string]*openFile)
|
||||
)
|
||||
|
||||
type openFile struct {
|
||||
wkfs.File
|
||||
path string // map key of openFiles
|
||||
refCount int
|
||||
}
|
||||
|
||||
type openFileHandle struct {
|
||||
closed bool
|
||||
*openFile
|
||||
}
|
||||
|
||||
func (f *openFileHandle) Close() error {
|
||||
openFileMu.Lock()
|
||||
if f.closed {
|
||||
openFileMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
f.closed = true
|
||||
f.refCount--
|
||||
if f.refCount < 0 {
|
||||
panic("unexpected negative refcount")
|
||||
}
|
||||
zero := f.refCount == 0
|
||||
if zero {
|
||||
delete(openFiles, f.path)
|
||||
}
|
||||
openFileMu.Unlock()
|
||||
if !zero {
|
||||
return nil
|
||||
}
|
||||
return f.openFile.File.Close()
|
||||
}
|
||||
|
||||
// Open opens the given file path for reading, reusing existing file descriptors
|
||||
// when possible.
|
||||
func Open(path string) (readerutil.ReaderAtCloser, error) {
|
||||
openFileMu.Lock()
|
||||
of := openFiles[path]
|
||||
if of != nil {
|
||||
of.refCount++
|
||||
openFileMu.Unlock()
|
||||
return &openFileHandle{false, of}, nil
|
||||
}
|
||||
openFileMu.Unlock() // release the lock while we call os.Open
|
||||
|
||||
winner := false // this goroutine made it into Do's func
|
||||
|
||||
// Returns an *openFile
|
||||
resi, err := openerGroup.Do(path, func() (interface{}, error) {
|
||||
winner = true
|
||||
f, err := wkfs.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
of := &openFile{
|
||||
File: f,
|
||||
path: path,
|
||||
refCount: 1,
|
||||
}
|
||||
openFileMu.Lock()
|
||||
openFiles[path] = of
|
||||
openFileMu.Unlock()
|
||||
return of, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
of = resi.(*openFile)
|
||||
|
||||
// If our os.Open was dup-suppressed, we have to increment our
|
||||
// reference count.
|
||||
if !winner {
|
||||
openFileMu.Lock()
|
||||
if of.refCount == 0 {
|
||||
// Winner already closed it. Try again (rare).
|
||||
openFileMu.Unlock()
|
||||
return Open(path)
|
||||
}
|
||||
of.refCount++
|
||||
openFileMu.Unlock()
|
||||
}
|
||||
return &openFileHandle{false, of}, nil
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Go4 Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package singlereader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenSingle(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping in short mode")
|
||||
}
|
||||
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4))
|
||||
f, err := ioutil.TempFile("", "foo")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
contents := []byte("Some file contents")
|
||||
if _, err := f.Write(contents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
|
||||
const j = 4
|
||||
errc := make(chan error, j)
|
||||
for i := 1; i < j; i++ {
|
||||
go func() {
|
||||
buf := make([]byte, len(contents))
|
||||
for i := 0; i < 400; i++ {
|
||||
rac, err := Open(f.Name())
|
||||
if err != nil {
|
||||
errc <- err
|
||||
return
|
||||
}
|
||||
n, err := rac.ReadAt(buf, 0)
|
||||
if err != nil {
|
||||
errc <- err
|
||||
return
|
||||
}
|
||||
if n != len(contents) || !bytes.Equal(buf, contents) {
|
||||
errc <- fmt.Errorf("read %d, %q; want %d, %q", n, buf, len(contents), contents)
|
||||
return
|
||||
}
|
||||
if err := rac.Close(); err != nil {
|
||||
errc <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
errc <- nil
|
||||
}()
|
||||
}
|
||||
for i := 1; i < j; i++ {
|
||||
if err := <-errc; err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build go1.6
|
||||
// +build arm
|
||||
|
||||
#include "textflag.h"
|
||||
#include "funcdata.h"
|
||||
|
||||
// func typedmemmove(reflect_rtype, src unsafe.Pointer, size uintptr)
|
||||
TEXT ·typedmemmove(SB),(NOSPLIT|WRAPPER),$0-24
|
||||
B runtime·typedmemmove(SB)
|
||||
|
||||
// func memmove(dst, src unsafe.Pointer, size uintptr)
|
||||
TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24
|
||||
B runtime·memmove(SB)
|
||||
@ -1,13 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !go1.5
|
||||
// +build arm
|
||||
|
||||
#include "textflag.h"
|
||||
#include "funcdata.h"
|
||||
|
||||
// func memmove(dst, src unsafe.Pointer, size uintptr)
|
||||
TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24
|
||||
B runtime·memmove(SB)
|
||||
@ -1,17 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build go1.5,!js,!safe,!appengine
|
||||
// +build amd64 386
|
||||
|
||||
#include "textflag.h"
|
||||
#include "funcdata.h"
|
||||
|
||||
// func typedmemmove(reflect_rtype, src unsafe.Pointer, size uintptr)
|
||||
TEXT ·typedmemmove(SB),(NOSPLIT|WRAPPER),$0-24
|
||||
JMP runtime·typedmemmove(SB)
|
||||
|
||||
// func memmove(dst, src unsafe.Pointer, size uintptr)
|
||||
TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24
|
||||
JMP runtime·memmove(SB)
|
||||
@ -1,13 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !go1.5,!js,!safe,!appengine
|
||||
// +build amd64 386
|
||||
|
||||
#include "textflag.h"
|
||||
#include "funcdata.h"
|
||||
|
||||
// func memmove(dst, src unsafe.Pointer, size uintptr)
|
||||
TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24
|
||||
JMP runtime·memmove(SB)
|
||||
@ -1,40 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package reflectutil contains reflect utilities.
|
||||
package reflectutil
|
||||
|
||||
import "reflect"
|
||||
|
||||
// hasPointers reports whether the given type contains any pointers,
|
||||
// including any internal pointers in slices, funcs, maps, channels,
|
||||
// etc.
|
||||
//
|
||||
// This function exists for Swapper's internal use, instead of reaching
|
||||
// into the runtime's *reflect._rtype kind&kindNoPointers flag.
|
||||
func hasPointers(t reflect.Type) bool {
|
||||
if t == nil {
|
||||
panic("nil Type")
|
||||
}
|
||||
k := t.Kind()
|
||||
if k <= reflect.Complex128 {
|
||||
return false
|
||||
}
|
||||
switch k {
|
||||
default:
|
||||
// chan, func, interface, map, ptr, slice, string, unsafepointer
|
||||
// And anything else. It's safer to err on the side of true.
|
||||
return true
|
||||
case reflect.Array:
|
||||
return hasPointers(t.Elem())
|
||||
case reflect.Struct:
|
||||
num := t.NumField()
|
||||
for i := 0; i < num; i++ {
|
||||
if hasPointers(t.Field(i).Type) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@ -1,71 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package reflectutil
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHasPointers(t *testing.T) {
|
||||
tests := []struct {
|
||||
val interface{}
|
||||
want bool
|
||||
}{
|
||||
{false, false},
|
||||
{int(1), false},
|
||||
{int8(1), false},
|
||||
{int16(1), false},
|
||||
{int32(1), false},
|
||||
{int64(1), false},
|
||||
{uint(1), false},
|
||||
{uint8(1), false},
|
||||
{uint16(1), false},
|
||||
{uint32(1), false},
|
||||
{uint64(1), false},
|
||||
{uintptr(1), false},
|
||||
{float32(1.0), false},
|
||||
{float64(1.0), false},
|
||||
{complex64(1.0i), false},
|
||||
{complex128(1.0i), false},
|
||||
|
||||
{[...]int{1, 2}, false},
|
||||
{[...]*int{nil, nil}, true},
|
||||
|
||||
{make(chan bool), true},
|
||||
|
||||
{TestHasPointers, true},
|
||||
|
||||
{map[string]string{"foo": "bar"}, true},
|
||||
|
||||
{new(int), true},
|
||||
|
||||
{[]int{1, 2}, true},
|
||||
|
||||
{"foo", true},
|
||||
|
||||
{struct{}{}, false},
|
||||
{struct{ int }{0}, false},
|
||||
{struct {
|
||||
a int
|
||||
b bool
|
||||
}{0, false}, false},
|
||||
{struct {
|
||||
a int
|
||||
b string
|
||||
}{0, ""}, true},
|
||||
{struct{ *int }{nil}, true},
|
||||
{struct {
|
||||
a *int
|
||||
b int
|
||||
}{nil, 0}, true},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
got := hasPointers(reflect.TypeOf(tt.val))
|
||||
if got != tt.want {
|
||||
t.Errorf("%d. hasPointers(%T) = %v; want %v", i, tt.val, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package reflectutil
|
||||
|
||||
import "reflect"
|
||||
|
||||
// Swapper returns a function which swaps the elements in slice.
|
||||
// Swapper panics if the provided interface is not a slice.
|
||||
//
|
||||
// Its goal is to work safely and efficiently for all versions and
|
||||
// variants of Go: pre-Go1.5, Go1.5+, safe, unsafe, App Engine,
|
||||
// GopherJS, etc.
|
||||
func Swapper(slice interface{}) func(i, j int) {
|
||||
v := reflect.ValueOf(slice)
|
||||
if v.Kind() != reflect.Slice {
|
||||
panic(&reflect.ValueError{Method: "reflectutil.Swapper", Kind: v.Kind()})
|
||||
}
|
||||
return swapper(v)
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build js appengine safe ppc64 ppc64le arm64 mips mipsle mips64 mips64le
|
||||
|
||||
package reflectutil
|
||||
|
||||
import "reflect"
|
||||
|
||||
func swapper(slice reflect.Value) func(i, j int) {
|
||||
tmp := reflect.New(slice.Type().Elem()).Elem()
|
||||
return func(i, j int) {
|
||||
v1 := slice.Index(i)
|
||||
v2 := slice.Index(j)
|
||||
tmp.Set(v1)
|
||||
v1.Set(v2)
|
||||
v2.Set(tmp)
|
||||
}
|
||||
}
|
||||
@ -1,110 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package reflectutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSwapper(t *testing.T) {
|
||||
type I int
|
||||
var a, b, c I
|
||||
type pair struct {
|
||||
x, y int
|
||||
}
|
||||
type pairPtr struct {
|
||||
x, y int
|
||||
p *I
|
||||
}
|
||||
type S string
|
||||
|
||||
tests := []struct {
|
||||
in interface{}
|
||||
i, j int
|
||||
want interface{}
|
||||
}{
|
||||
{
|
||||
in: []int{1, 20, 300},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []int{300, 20, 1},
|
||||
},
|
||||
{
|
||||
in: []uintptr{1, 20, 300},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []uintptr{300, 20, 1},
|
||||
},
|
||||
{
|
||||
in: []int16{1, 20, 300},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []int16{300, 20, 1},
|
||||
},
|
||||
{
|
||||
in: []int8{1, 20, 100},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []int8{100, 20, 1},
|
||||
},
|
||||
{
|
||||
in: []*I{&a, &b, &c},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []*I{&c, &b, &a},
|
||||
},
|
||||
{
|
||||
in: []string{"eric", "sergey", "larry"},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []string{"larry", "sergey", "eric"},
|
||||
},
|
||||
{
|
||||
in: []S{"eric", "sergey", "larry"},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []S{"larry", "sergey", "eric"},
|
||||
},
|
||||
{
|
||||
in: []pair{{1, 2}, {3, 4}, {5, 6}},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []pair{{5, 6}, {3, 4}, {1, 2}},
|
||||
},
|
||||
{
|
||||
in: []pairPtr{{1, 2, &a}, {3, 4, &b}, {5, 6, &c}},
|
||||
i: 0,
|
||||
j: 2,
|
||||
want: []pairPtr{{5, 6, &c}, {3, 4, &b}, {1, 2, &a}},
|
||||
},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
inStr := fmt.Sprint(tt.in)
|
||||
Swapper(tt.in)(tt.i, tt.j)
|
||||
if !reflect.DeepEqual(tt.in, tt.want) {
|
||||
t.Errorf("%d. swapping %v and %v of %v = %v; want %v", i, tt.i, tt.j, inStr, tt.in, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSwap(b *testing.B) {
|
||||
const N = 1024
|
||||
strs := make([]string, N)
|
||||
for i := range strs {
|
||||
strs[i] = strconv.Itoa(i)
|
||||
}
|
||||
swap := Swapper(strs)
|
||||
|
||||
b.ResetTimer()
|
||||
i, j := 0, 1
|
||||
for n := 0; n < b.N; n++ {
|
||||
i = (i + 1) % N
|
||||
j = (j + 2) % N
|
||||
swap(i, j)
|
||||
}
|
||||
}
|
||||
@ -1,106 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !ppc64,!ppc64le,!arm64,!mips,!mipsle,!mips64,!mips64le
|
||||
// +build !js,!appengine,!safe
|
||||
|
||||
package reflectutil
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const
|
||||
|
||||
// arrayAt returns the i-th element of p, a C-array whose elements are
|
||||
// eltSize wide (in bytes).
|
||||
func arrayAt(p unsafe.Pointer, i int, eltSize uintptr) unsafe.Pointer {
|
||||
return unsafe.Pointer(uintptr(p) + uintptr(i)*eltSize)
|
||||
}
|
||||
|
||||
type sliceHeader struct {
|
||||
Data unsafe.Pointer
|
||||
Len int
|
||||
Cap int
|
||||
}
|
||||
|
||||
func swapper(v reflect.Value) func(i, j int) {
|
||||
maxLen := uint(v.Len())
|
||||
|
||||
s := sliceHeader{unsafe.Pointer(v.Pointer()), int(maxLen), int(maxLen)}
|
||||
tt := v.Type()
|
||||
elemt := tt.Elem()
|
||||
typ := unsafe.Pointer(reflect.ValueOf(elemt).Pointer())
|
||||
|
||||
size := elemt.Size()
|
||||
hasPtr := hasPointers(elemt)
|
||||
|
||||
// Some common & small cases, without using memmove:
|
||||
if hasPtr {
|
||||
if size == ptrSize {
|
||||
var ps []unsafe.Pointer
|
||||
*(*sliceHeader)(unsafe.Pointer(&ps)) = s
|
||||
return func(i, j int) { ps[i], ps[j] = ps[j], ps[i] }
|
||||
}
|
||||
if elemt.Kind() == reflect.String {
|
||||
var ss []string
|
||||
*(*sliceHeader)(unsafe.Pointer(&ss)) = s
|
||||
return func(i, j int) { ss[i], ss[j] = ss[j], ss[i] }
|
||||
}
|
||||
} else {
|
||||
switch size {
|
||||
case 8:
|
||||
var is []int64
|
||||
*(*sliceHeader)(unsafe.Pointer(&is)) = s
|
||||
return func(i, j int) { is[i], is[j] = is[j], is[i] }
|
||||
case 4:
|
||||
var is []int32
|
||||
*(*sliceHeader)(unsafe.Pointer(&is)) = s
|
||||
return func(i, j int) { is[i], is[j] = is[j], is[i] }
|
||||
case 2:
|
||||
var is []int16
|
||||
*(*sliceHeader)(unsafe.Pointer(&is)) = s
|
||||
return func(i, j int) { is[i], is[j] = is[j], is[i] }
|
||||
case 1:
|
||||
var is []int8
|
||||
*(*sliceHeader)(unsafe.Pointer(&is)) = s
|
||||
return func(i, j int) { is[i], is[j] = is[j], is[i] }
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate scratch space for swaps:
|
||||
tmpVal := reflect.New(elemt)
|
||||
tmp := unsafe.Pointer(tmpVal.Pointer())
|
||||
|
||||
// If no pointers (or Go 1.4 or below), we don't require typedmemmove:
|
||||
if !haveTypedMemmove || !hasPtr {
|
||||
return func(i, j int) {
|
||||
if uint(i) >= maxLen || uint(j) >= maxLen {
|
||||
panic("reflect: slice index out of range")
|
||||
}
|
||||
val1 := arrayAt(s.Data, i, size)
|
||||
val2 := arrayAt(s.Data, j, size)
|
||||
memmove(tmp, val1, size)
|
||||
memmove(val1, val2, size)
|
||||
memmove(val2, tmp, size)
|
||||
}
|
||||
}
|
||||
|
||||
return func(i, j int) {
|
||||
if uint(i) >= maxLen || uint(j) >= maxLen {
|
||||
panic("reflect: slice index out of range")
|
||||
}
|
||||
val1 := arrayAt(s.Data, i, size)
|
||||
val2 := arrayAt(s.Data, j, size)
|
||||
typedmemmove(typ, tmp, val1)
|
||||
typedmemmove(typ, val1, val2)
|
||||
typedmemmove(typ, val2, tmp)
|
||||
}
|
||||
}
|
||||
|
||||
// memmove copies size bytes from src to dst.
|
||||
// The memory must not contain any pointers.
|
||||
//go:noescape
|
||||
func memmove(dst, src unsafe.Pointer, size uintptr)
|
||||
@ -1,16 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !ppc64,!ppc64le,!arm64,!mips,!mipsle,!mips64,!mips64le
|
||||
// +build !go1.5,!js,!appengine,!safe
|
||||
|
||||
package reflectutil
|
||||
|
||||
import "unsafe"
|
||||
|
||||
const haveTypedMemmove = false
|
||||
|
||||
func typedmemmove(reflect_rtype, dst, src unsafe.Pointer) {
|
||||
panic("never called") // only here so swapper_unsafe.go compiles
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !ppc64,!ppc64le,!arm64,!mips,!mipsle,!mips64,!mips64le
|
||||
// +build go1.5,!js,!appengine,!safe
|
||||
|
||||
package reflectutil
|
||||
|
||||
import "unsafe"
|
||||
|
||||
const haveTypedMemmove = true
|
||||
|
||||
// typedmemmove copies a value of type t to dst from src.
|
||||
//go:noescape
|
||||
func typedmemmove(reflect_rtype, dst, src unsafe.Pointer)
|
||||
@ -1,45 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"go4.org/sort"
|
||||
)
|
||||
|
||||
type Person struct {
|
||||
Name string
|
||||
Age int
|
||||
}
|
||||
|
||||
func (p Person) String() string {
|
||||
return fmt.Sprintf("%s: %d", p.Name, p.Age)
|
||||
}
|
||||
|
||||
// ByAge implements sort.Interface for []Person based on
|
||||
// the Age field.
|
||||
type ByAge []Person
|
||||
|
||||
func (a ByAge) Len() int { return len(a) }
|
||||
func (a ByAge) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a ByAge) Less(i, j int) bool { return a[i].Age < a[j].Age }
|
||||
|
||||
func ExampleSort() {
|
||||
people := []Person{
|
||||
{"Bob", 31},
|
||||
{"John", 42},
|
||||
{"Michael", 17},
|
||||
{"Jenny", 26},
|
||||
}
|
||||
|
||||
fmt.Println(people)
|
||||
sort.Sort(ByAge(people))
|
||||
fmt.Println(people)
|
||||
|
||||
// Output:
|
||||
// [Bob: 31 John: 42 Michael: 17 Jenny: 26]
|
||||
// [Michael: 17 Jenny: 26 Bob: 31 John: 42]
|
||||
}
|
||||
@ -1,96 +0,0 @@
|
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// A couple of type definitions to make the units clear.
|
||||
type earthMass float64
|
||||
type au float64
|
||||
|
||||
// A Planet defines the properties of a solar system object.
|
||||
type Planet struct {
|
||||
name string
|
||||
mass earthMass
|
||||
distance au
|
||||
}
|
||||
|
||||
// By is the type of a "less" function that defines the ordering of its Planet arguments.
|
||||
type By func(p1, p2 *Planet) bool
|
||||
|
||||
// Sort is a method on the function type, By, that sorts the argument slice according to the function.
|
||||
func (by By) Sort(planets []Planet) {
|
||||
ps := &planetSorter{
|
||||
planets: planets,
|
||||
by: by, // The Sort method's receiver is the function (closure) that defines the sort order.
|
||||
}
|
||||
sort.Sort(ps)
|
||||
}
|
||||
|
||||
// planetSorter joins a By function and a slice of Planets to be sorted.
|
||||
type planetSorter struct {
|
||||
planets []Planet
|
||||
by func(p1, p2 *Planet) bool // Closure used in the Less method.
|
||||
}
|
||||
|
||||
// Len is part of sort.Interface.
|
||||
func (s *planetSorter) Len() int {
|
||||
return len(s.planets)
|
||||
}
|
||||
|
||||
// Swap is part of sort.Interface.
|
||||
func (s *planetSorter) Swap(i, j int) {
|
||||
s.planets[i], s.planets[j] = s.planets[j], s.planets[i]
|
||||
}
|
||||
|
||||
// Less is part of sort.Interface. It is implemented by calling the "by" closure in the sorter.
|
||||
func (s *planetSorter) Less(i, j int) bool {
|
||||
return s.by(&s.planets[i], &s.planets[j])
|
||||
}
|
||||
|
||||
var planets = []Planet{
|
||||
{"Mercury", 0.055, 0.4},
|
||||
{"Venus", 0.815, 0.7},
|
||||
{"Earth", 1.0, 1.0},
|
||||
{"Mars", 0.107, 1.5},
|
||||
}
|
||||
|
||||
// ExampleSortKeys demonstrates a technique for sorting a struct type using programmable sort criteria.
|
||||
func Example_sortKeys() {
|
||||
// Closures that order the Planet structure.
|
||||
name := func(p1, p2 *Planet) bool {
|
||||
return p1.name < p2.name
|
||||
}
|
||||
mass := func(p1, p2 *Planet) bool {
|
||||
return p1.mass < p2.mass
|
||||
}
|
||||
distance := func(p1, p2 *Planet) bool {
|
||||
return p1.distance < p2.distance
|
||||
}
|
||||
decreasingDistance := func(p1, p2 *Planet) bool {
|
||||
return !distance(p1, p2)
|
||||
}
|
||||
|
||||
// Sort the planets by the various criteria.
|
||||
By(name).Sort(planets)
|
||||
fmt.Println("By name:", planets)
|
||||
|
||||
By(mass).Sort(planets)
|
||||
fmt.Println("By mass:", planets)
|
||||
|
||||
By(distance).Sort(planets)
|
||||
fmt.Println("By distance:", planets)
|
||||
|
||||
By(decreasingDistance).Sort(planets)
|
||||
fmt.Println("By decreasing distance:", planets)
|
||||
|
||||
// Output: By name: [{Earth 1 1} {Mars 0.107 1.5} {Mercury 0.055 0.4} {Venus 0.815 0.7}]
|
||||
// By mass: [{Mercury 0.055 0.4} {Mars 0.107 1.5} {Venus 0.815 0.7} {Earth 1 1}]
|
||||
// By distance: [{Mercury 0.055 0.4} {Venus 0.815 0.7} {Earth 1 1} {Mars 0.107 1.5}]
|
||||
// By decreasing distance: [{Mars 0.107 1.5} {Earth 1 1} {Venus 0.815 0.7} {Mercury 0.055 0.4}]
|
||||
}
|
||||
@ -1,133 +0,0 @@
|
||||
// +build go1.6
|
||||
|
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// A Change is a record of source code changes, recording user, language, and delta size.
|
||||
type Change struct {
|
||||
user string
|
||||
language string
|
||||
lines int
|
||||
}
|
||||
|
||||
type lessFunc func(p1, p2 *Change) bool
|
||||
|
||||
// multiSorter implements the Sort interface, sorting the changes within.
|
||||
type multiSorter struct {
|
||||
changes []Change
|
||||
less []lessFunc
|
||||
}
|
||||
|
||||
// Sort sorts the argument slice according to the less functions passed to OrderedBy.
|
||||
func (ms *multiSorter) Sort(changes []Change) {
|
||||
ms.changes = changes
|
||||
sort.Sort(ms)
|
||||
}
|
||||
|
||||
// OrderedBy returns a Sorter that sorts using the less functions, in order.
|
||||
// Call its Sort method to sort the data.
|
||||
func OrderedBy(less ...lessFunc) *multiSorter {
|
||||
return &multiSorter{
|
||||
less: less,
|
||||
}
|
||||
}
|
||||
|
||||
// Len is part of sort.Interface.
|
||||
func (ms *multiSorter) Len() int {
|
||||
return len(ms.changes)
|
||||
}
|
||||
|
||||
// Swap is part of sort.Interface.
|
||||
func (ms *multiSorter) Swap(i, j int) {
|
||||
ms.changes[i], ms.changes[j] = ms.changes[j], ms.changes[i]
|
||||
}
|
||||
|
||||
// Less is part of sort.Interface. It is implemented by looping along the
|
||||
// less functions until it finds a comparison that is either Less or
|
||||
// !Less. Note that it can call the less functions twice per call. We
|
||||
// could change the functions to return -1, 0, 1 and reduce the
|
||||
// number of calls for greater efficiency: an exercise for the reader.
|
||||
func (ms *multiSorter) Less(i, j int) bool {
|
||||
p, q := &ms.changes[i], &ms.changes[j]
|
||||
// Try all but the last comparison.
|
||||
var k int
|
||||
for k = 0; k < len(ms.less)-1; k++ {
|
||||
less := ms.less[k]
|
||||
switch {
|
||||
case less(p, q):
|
||||
// p < q, so we have a decision.
|
||||
return true
|
||||
case less(q, p):
|
||||
// p > q, so we have a decision.
|
||||
return false
|
||||
}
|
||||
// p == q; try the next comparison.
|
||||
}
|
||||
// All comparisons to here said "equal", so just return whatever
|
||||
// the final comparison reports.
|
||||
return ms.less[k](p, q)
|
||||
}
|
||||
|
||||
var changes = []Change{
|
||||
{"gri", "Go", 100},
|
||||
{"ken", "C", 150},
|
||||
{"glenda", "Go", 200},
|
||||
{"rsc", "Go", 200},
|
||||
{"r", "Go", 100},
|
||||
{"ken", "Go", 200},
|
||||
{"dmr", "C", 100},
|
||||
{"r", "C", 150},
|
||||
{"gri", "Smalltalk", 80},
|
||||
}
|
||||
|
||||
// ExampleMultiKeys demonstrates a technique for sorting a struct type using different
|
||||
// sets of multiple fields in the comparison. We chain together "Less" functions, each of
|
||||
// which compares a single field.
|
||||
func Example_sortMultiKeys() {
|
||||
// Closures that order the Change structure.
|
||||
user := func(c1, c2 *Change) bool {
|
||||
return c1.user < c2.user
|
||||
}
|
||||
language := func(c1, c2 *Change) bool {
|
||||
return c1.language < c2.language
|
||||
}
|
||||
increasingLines := func(c1, c2 *Change) bool {
|
||||
return c1.lines < c2.lines
|
||||
}
|
||||
decreasingLines := func(c1, c2 *Change) bool {
|
||||
return c1.lines > c2.lines // Note: > orders downwards.
|
||||
}
|
||||
|
||||
// Simple use: Sort by user.
|
||||
OrderedBy(user).Sort(changes)
|
||||
fmt.Println("By user:", changes)
|
||||
|
||||
// More examples.
|
||||
OrderedBy(user, increasingLines).Sort(changes)
|
||||
fmt.Println("By user,<lines:", changes)
|
||||
|
||||
OrderedBy(user, decreasingLines).Sort(changes)
|
||||
fmt.Println("By user,>lines:", changes)
|
||||
|
||||
OrderedBy(language, increasingLines).Sort(changes)
|
||||
fmt.Println("By language,<lines:", changes)
|
||||
|
||||
OrderedBy(language, increasingLines, user).Sort(changes)
|
||||
fmt.Println("By language,<lines,user:", changes)
|
||||
|
||||
// Output:
|
||||
// By user: [{dmr C 100} {glenda Go 200} {gri Go 100} {gri Smalltalk 80} {ken C 150} {ken Go 200} {r Go 100} {r C 150} {rsc Go 200}]
|
||||
// By user,<lines: [{dmr C 100} {glenda Go 200} {gri Smalltalk 80} {gri Go 100} {ken C 150} {ken Go 200} {r Go 100} {r C 150} {rsc Go 200}]
|
||||
// By user,>lines: [{dmr C 100} {glenda Go 200} {gri Go 100} {gri Smalltalk 80} {ken Go 200} {ken C 150} {r C 150} {r Go 100} {rsc Go 200}]
|
||||
// By language,<lines: [{dmr C 100} {ken C 150} {r C 150} {r Go 100} {gri Go 100} {ken Go 200} {glenda Go 200} {rsc Go 200} {gri Smalltalk 80}]
|
||||
// By language,<lines,user: [{dmr C 100} {ken C 150} {r C 150} {gri Go 100} {r Go 100} {glenda Go 200} {ken Go 200} {rsc Go 200} {gri Smalltalk 80}]
|
||||
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"go4.org/sort"
|
||||
)
|
||||
|
||||
func Example() {
|
||||
people := []Person{
|
||||
{Name: "Bob", Age: 31},
|
||||
{Name: "John", Age: 42},
|
||||
{Name: "Michael", Age: 17},
|
||||
{Name: "Jenny", Age: 26},
|
||||
}
|
||||
|
||||
fmt.Println(people)
|
||||
sort.Slice(people, func(i, j int) bool { return people[i].Age < people[j].Age })
|
||||
fmt.Println(people)
|
||||
|
||||
// Output:
|
||||
// [Bob: 31 John: 42 Michael: 17 Jenny: 26]
|
||||
// [Michael: 17 Jenny: 26 Bob: 31 John: 42]
|
||||
}
|
||||
|
||||
func ExampleSlice() {
|
||||
people := []Person{
|
||||
{Name: "Bob", Age: 31},
|
||||
{Name: "John", Age: 42},
|
||||
{Name: "Michael", Age: 17},
|
||||
{Name: "Jenny", Age: 26},
|
||||
}
|
||||
|
||||
fmt.Println(people)
|
||||
sort.Slice(people, func(i, j int) bool { return people[i].Age < people[j].Age })
|
||||
fmt.Println(people)
|
||||
|
||||
// Output:
|
||||
// [Bob: 31 John: 42 Michael: 17 Jenny: 26]
|
||||
// [Michael: 17 Jenny: 26 Bob: 31 John: 42]
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
func ExampleInts() {
|
||||
s := []int{5, 2, 6, 3, 1, 4} // unsorted
|
||||
sort.Ints(s)
|
||||
fmt.Println(s)
|
||||
// Output: [1 2 3 4 5 6]
|
||||
}
|
||||
|
||||
func ExampleReverse() {
|
||||
s := []int{5, 2, 6, 3, 1, 4} // unsorted
|
||||
sort.Sort(sort.Reverse(sort.IntSlice(s)))
|
||||
fmt.Println(s)
|
||||
// Output: [6 5 4 3 2 1]
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type Grams int
|
||||
|
||||
func (g Grams) String() string { return fmt.Sprintf("%dg", int(g)) }
|
||||
|
||||
type Organ struct {
|
||||
Name string
|
||||
Weight Grams
|
||||
}
|
||||
|
||||
type Organs []*Organ
|
||||
|
||||
func (s Organs) Len() int { return len(s) }
|
||||
func (s Organs) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
// ByName implements sort.Interface by providing Less and using the Len and
|
||||
// Swap methods of the embedded Organs value.
|
||||
type ByName struct{ Organs }
|
||||
|
||||
func (s ByName) Less(i, j int) bool { return s.Organs[i].Name < s.Organs[j].Name }
|
||||
|
||||
// ByWeight implements sort.Interface by providing Less and using the Len and
|
||||
// Swap methods of the embedded Organs value.
|
||||
type ByWeight struct{ Organs }
|
||||
|
||||
func (s ByWeight) Less(i, j int) bool { return s.Organs[i].Weight < s.Organs[j].Weight }
|
||||
|
||||
func Example_sortWrapper() {
|
||||
s := []*Organ{
|
||||
{"brain", 1340},
|
||||
{"heart", 290},
|
||||
{"liver", 1494},
|
||||
{"pancreas", 131},
|
||||
{"prostate", 62},
|
||||
{"spleen", 162},
|
||||
}
|
||||
|
||||
sort.Sort(ByWeight{s})
|
||||
fmt.Println("Organs by weight:")
|
||||
printOrgans(s)
|
||||
|
||||
sort.Sort(ByName{s})
|
||||
fmt.Println("Organs by name:")
|
||||
printOrgans(s)
|
||||
|
||||
// Output:
|
||||
// Organs by weight:
|
||||
// prostate (62g)
|
||||
// pancreas (131g)
|
||||
// spleen (162g)
|
||||
// heart (290g)
|
||||
// brain (1340g)
|
||||
// liver (1494g)
|
||||
// Organs by name:
|
||||
// brain (1340g)
|
||||
// heart (290g)
|
||||
// liver (1494g)
|
||||
// pancreas (131g)
|
||||
// prostate (62g)
|
||||
// spleen (162g)
|
||||
}
|
||||
|
||||
func printOrgans(s []*Organ) {
|
||||
for _, o := range s {
|
||||
fmt.Printf("%-8s (%v)\n", o.Name, o.Weight)
|
||||
}
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort
|
||||
|
||||
func Heapsort(data Interface) {
|
||||
heapSort(data, 0, data.Len())
|
||||
}
|
||||
@ -1,122 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// This program is run via "go generate" (via a directive in sort.go)
|
||||
// to generate zfuncversion.go.
|
||||
//
|
||||
// It copies sort.go to zfuncversion.go, only retaining funcs which
|
||||
// take a "data Interface" parameter, and renaming each to have a
|
||||
// "_func" suffix and taking a "data lessSwap" instead. It then rewrites
|
||||
// each internal function call to the appropriate _func variants.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"go/ast"
|
||||
"go/format"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
var fset = token.NewFileSet()
|
||||
|
||||
func main() {
|
||||
af, err := parser.ParseFile(fset, "sort.go", nil, 0)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
af.Doc = nil
|
||||
af.Imports = nil
|
||||
af.Comments = nil
|
||||
|
||||
var newDecl []ast.Decl
|
||||
for _, d := range af.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if fd.Recv != nil || fd.Name.IsExported() {
|
||||
continue
|
||||
}
|
||||
typ := fd.Type
|
||||
if len(typ.Params.List) < 1 {
|
||||
continue
|
||||
}
|
||||
arg0 := typ.Params.List[0]
|
||||
arg0Name := arg0.Names[0].Name
|
||||
arg0Type := arg0.Type.(*ast.Ident)
|
||||
if arg0Name != "data" || arg0Type.Name != "Interface" {
|
||||
continue
|
||||
}
|
||||
arg0Type.Name = "lessSwap"
|
||||
|
||||
newDecl = append(newDecl, fd)
|
||||
}
|
||||
af.Decls = newDecl
|
||||
ast.Walk(visitFunc(rewriteCalls), af)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := format.Node(&out, fset, af); err != nil {
|
||||
log.Fatalf("format.Node: %v", err)
|
||||
}
|
||||
|
||||
// Get rid of blank lines after removal of comments.
|
||||
src := regexp.MustCompile(`\n{2,}`).ReplaceAll(out.Bytes(), []byte("\n"))
|
||||
|
||||
// Add comments to each func, for the lost reader.
|
||||
// This is so much easier than adding comments via the AST
|
||||
// and trying to get position info correct.
|
||||
src = regexp.MustCompile(`(?m)^func (\w+)`).ReplaceAll(src, []byte("\n// Auto-generated variant of sort.go:$1\nfunc ${1}_func"))
|
||||
|
||||
// Final gofmt.
|
||||
src, err = format.Source(src)
|
||||
if err != nil {
|
||||
log.Fatalf("format.Source: %v on\n%s", err, src)
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
out.WriteString(`// DO NOT EDIT; AUTO-GENERATED from sort.go using genzfunc.go
|
||||
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
`)
|
||||
out.Write(src)
|
||||
|
||||
const target = "zfuncversion.go"
|
||||
if err := ioutil.WriteFile(target, out.Bytes(), 0644); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type visitFunc func(ast.Node) ast.Visitor
|
||||
|
||||
func (f visitFunc) Visit(n ast.Node) ast.Visitor { return f(n) }
|
||||
|
||||
func rewriteCalls(n ast.Node) ast.Visitor {
|
||||
ce, ok := n.(*ast.CallExpr)
|
||||
if ok {
|
||||
rewriteCall(ce)
|
||||
}
|
||||
return visitFunc(rewriteCalls)
|
||||
}
|
||||
|
||||
func rewriteCall(ce *ast.CallExpr) {
|
||||
ident, ok := ce.Fun.(*ast.Ident)
|
||||
if !ok {
|
||||
// e.g. skip SelectorExpr (data.Less(..) calls)
|
||||
return
|
||||
}
|
||||
if len(ce.Args) < 1 {
|
||||
return
|
||||
}
|
||||
ident.Name += "_func"
|
||||
}
|
||||
@ -1,112 +0,0 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// This file implements binary search.
|
||||
|
||||
package sort
|
||||
|
||||
// Search uses binary search to find and return the smallest index i
|
||||
// in [0, n) at which f(i) is true, assuming that on the range [0, n),
|
||||
// f(i) == true implies f(i+1) == true. That is, Search requires that
|
||||
// f is false for some (possibly empty) prefix of the input range [0, n)
|
||||
// and then true for the (possibly empty) remainder; Search returns
|
||||
// the first true index. If there is no such index, Search returns n.
|
||||
// (Note that the "not found" return value is not -1 as in, for instance,
|
||||
// strings.Index.)
|
||||
// Search calls f(i) only for i in the range [0, n).
|
||||
//
|
||||
// A common use of Search is to find the index i for a value x in
|
||||
// a sorted, indexable data structure such as an array or slice.
|
||||
// In this case, the argument f, typically a closure, captures the value
|
||||
// to be searched for, and how the data structure is indexed and
|
||||
// ordered.
|
||||
//
|
||||
// For instance, given a slice data sorted in ascending order,
|
||||
// the call Search(len(data), func(i int) bool { return data[i] >= 23 })
|
||||
// returns the smallest index i such that data[i] >= 23. If the caller
|
||||
// wants to find whether 23 is in the slice, it must test data[i] == 23
|
||||
// separately.
|
||||
//
|
||||
// Searching data sorted in descending order would use the <=
|
||||
// operator instead of the >= operator.
|
||||
//
|
||||
// To complete the example above, the following code tries to find the value
|
||||
// x in an integer slice data sorted in ascending order:
|
||||
//
|
||||
// x := 23
|
||||
// i := sort.Search(len(data), func(i int) bool { return data[i] >= x })
|
||||
// if i < len(data) && data[i] == x {
|
||||
// // x is present at data[i]
|
||||
// } else {
|
||||
// // x is not present in data,
|
||||
// // but i is the index where it would be inserted.
|
||||
// }
|
||||
//
|
||||
// As a more whimsical example, this program guesses your number:
|
||||
//
|
||||
// func GuessingGame() {
|
||||
// var s string
|
||||
// fmt.Printf("Pick an integer from 0 to 100.\n")
|
||||
// answer := sort.Search(100, func(i int) bool {
|
||||
// fmt.Printf("Is your number <= %d? ", i)
|
||||
// fmt.Scanf("%s", &s)
|
||||
// return s != "" && s[0] == 'y'
|
||||
// })
|
||||
// fmt.Printf("Your number is %d.\n", answer)
|
||||
// }
|
||||
//
|
||||
func Search(n int, f func(int) bool) int {
|
||||
// Define f(-1) == false and f(n) == true.
|
||||
// Invariant: f(i-1) == false, f(j) == true.
|
||||
i, j := 0, n
|
||||
for i < j {
|
||||
h := i + (j-i)/2 // avoid overflow when computing h
|
||||
// i ≤ h < j
|
||||
if !f(h) {
|
||||
i = h + 1 // preserves f(i-1) == false
|
||||
} else {
|
||||
j = h // preserves f(j) == true
|
||||
}
|
||||
}
|
||||
// i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i.
|
||||
return i
|
||||
}
|
||||
|
||||
// Convenience wrappers for common cases.
|
||||
|
||||
// SearchInts searches for x in a sorted slice of ints and returns the index
|
||||
// as specified by Search. The return value is the index to insert x if x is
|
||||
// not present (it could be len(a)).
|
||||
// The slice must be sorted in ascending order.
|
||||
//
|
||||
func SearchInts(a []int, x int) int {
|
||||
return Search(len(a), func(i int) bool { return a[i] >= x })
|
||||
}
|
||||
|
||||
// SearchFloat64s searches for x in a sorted slice of float64s and returns the index
|
||||
// as specified by Search. The return value is the index to insert x if x is not
|
||||
// present (it could be len(a)).
|
||||
// The slice must be sorted in ascending order.
|
||||
//
|
||||
func SearchFloat64s(a []float64, x float64) int {
|
||||
return Search(len(a), func(i int) bool { return a[i] >= x })
|
||||
}
|
||||
|
||||
// SearchStrings searches for x in a sorted slice of strings and returns the index
|
||||
// as specified by Search. The return value is the index to insert x if x is not
|
||||
// present (it could be len(a)).
|
||||
// The slice must be sorted in ascending order.
|
||||
//
|
||||
func SearchStrings(a []string, x string) int {
|
||||
return Search(len(a), func(i int) bool { return a[i] >= x })
|
||||
}
|
||||
|
||||
// Search returns the result of applying SearchInts to the receiver and x.
|
||||
func (p IntSlice) Search(x int) int { return SearchInts(p, x) }
|
||||
|
||||
// Search returns the result of applying SearchFloat64s to the receiver and x.
|
||||
func (p Float64Slice) Search(x float64) int { return SearchFloat64s(p, x) }
|
||||
|
||||
// Search returns the result of applying SearchStrings to the receiver and x.
|
||||
func (p StringSlice) Search(x string) int { return SearchStrings(p, x) }
|
||||
@ -1,161 +0,0 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
. "sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func f(a []int, x int) func(int) bool {
|
||||
return func(i int) bool {
|
||||
return a[i] >= x
|
||||
}
|
||||
}
|
||||
|
||||
var data = []int{0: -10, 1: -5, 2: 0, 3: 1, 4: 2, 5: 3, 6: 5, 7: 7, 8: 11, 9: 100, 10: 100, 11: 100, 12: 1000, 13: 10000}
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
n int
|
||||
f func(int) bool
|
||||
i int
|
||||
}{
|
||||
{"empty", 0, nil, 0},
|
||||
{"1 1", 1, func(i int) bool { return i >= 1 }, 1},
|
||||
{"1 true", 1, func(i int) bool { return true }, 0},
|
||||
{"1 false", 1, func(i int) bool { return false }, 1},
|
||||
{"1e9 991", 1e9, func(i int) bool { return i >= 991 }, 991},
|
||||
{"1e9 true", 1e9, func(i int) bool { return true }, 0},
|
||||
{"1e9 false", 1e9, func(i int) bool { return false }, 1e9},
|
||||
{"data -20", len(data), f(data, -20), 0},
|
||||
{"data -10", len(data), f(data, -10), 0},
|
||||
{"data -9", len(data), f(data, -9), 1},
|
||||
{"data -6", len(data), f(data, -6), 1},
|
||||
{"data -5", len(data), f(data, -5), 1},
|
||||
{"data 3", len(data), f(data, 3), 5},
|
||||
{"data 11", len(data), f(data, 11), 8},
|
||||
{"data 99", len(data), f(data, 99), 9},
|
||||
{"data 100", len(data), f(data, 100), 9},
|
||||
{"data 101", len(data), f(data, 101), 12},
|
||||
{"data 10000", len(data), f(data, 10000), 13},
|
||||
{"data 10001", len(data), f(data, 10001), 14},
|
||||
{"descending a", 7, func(i int) bool { return []int{99, 99, 59, 42, 7, 0, -1, -1}[i] <= 7 }, 4},
|
||||
{"descending 7", 1e9, func(i int) bool { return 1e9-i <= 7 }, 1e9 - 7},
|
||||
{"overflow", 2e9, func(i int) bool { return false }, 2e9},
|
||||
}
|
||||
|
||||
func TestSearch(t *testing.T) {
|
||||
for _, e := range tests {
|
||||
i := Search(e.n, e.f)
|
||||
if i != e.i {
|
||||
t.Errorf("%s: expected index %d; got %d", e.name, e.i, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// log2 computes the binary logarithm of x, rounded up to the next integer.
|
||||
// (log2(0) == 0, log2(1) == 0, log2(2) == 1, log2(3) == 2, etc.)
|
||||
//
|
||||
func log2(x int) int {
|
||||
n := 0
|
||||
for p := 1; p < x; p += p {
|
||||
// p == 2**n
|
||||
n++
|
||||
}
|
||||
// p/2 < x <= p == 2**n
|
||||
return n
|
||||
}
|
||||
|
||||
func TestSearchEfficiency(t *testing.T) {
|
||||
n := 100
|
||||
step := 1
|
||||
for exp := 2; exp < 10; exp++ {
|
||||
// n == 10**exp
|
||||
// step == 10**(exp-2)
|
||||
max := log2(n)
|
||||
for x := 0; x < n; x += step {
|
||||
count := 0
|
||||
i := Search(n, func(i int) bool { count++; return i >= x })
|
||||
if i != x {
|
||||
t.Errorf("n = %d: expected index %d; got %d", n, x, i)
|
||||
}
|
||||
if count > max {
|
||||
t.Errorf("n = %d, x = %d: expected <= %d calls; got %d", n, x, max, count)
|
||||
}
|
||||
}
|
||||
n *= 10
|
||||
step *= 10
|
||||
}
|
||||
}
|
||||
|
||||
// Smoke tests for convenience wrappers - not comprehensive.
|
||||
|
||||
var fdata = []float64{0: -3.14, 1: 0, 2: 1, 3: 2, 4: 1000.7}
|
||||
var sdata = []string{0: "f", 1: "foo", 2: "foobar", 3: "x"}
|
||||
|
||||
var wrappertests = []struct {
|
||||
name string
|
||||
result int
|
||||
i int
|
||||
}{
|
||||
{"SearchInts", SearchInts(data, 11), 8},
|
||||
{"SearchFloat64s", SearchFloat64s(fdata, 2.1), 4},
|
||||
{"SearchStrings", SearchStrings(sdata, ""), 0},
|
||||
{"IntSlice.Search", IntSlice(data).Search(0), 2},
|
||||
{"Float64Slice.Search", Float64Slice(fdata).Search(2.0), 3},
|
||||
{"StringSlice.Search", StringSlice(sdata).Search("x"), 3},
|
||||
}
|
||||
|
||||
func TestSearchWrappers(t *testing.T) {
|
||||
for _, e := range wrappertests {
|
||||
if e.result != e.i {
|
||||
t.Errorf("%s: expected index %d; got %d", e.name, e.i, e.result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runSearchWrappers() {
|
||||
SearchInts(data, 11)
|
||||
SearchFloat64s(fdata, 2.1)
|
||||
SearchStrings(sdata, "")
|
||||
IntSlice(data).Search(0)
|
||||
Float64Slice(fdata).Search(2.0)
|
||||
StringSlice(sdata).Search("x")
|
||||
}
|
||||
|
||||
func TestSearchWrappersDontAlloc(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping malloc count in short mode")
|
||||
}
|
||||
if runtime.GOMAXPROCS(0) > 1 {
|
||||
t.Skip("skipping; GOMAXPROCS>1")
|
||||
}
|
||||
allocs := testing.AllocsPerRun(100, runSearchWrappers)
|
||||
if allocs != 0 {
|
||||
t.Errorf("expected no allocs for runSearchWrappers, got %v", allocs)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSearchWrappers(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
runSearchWrappers()
|
||||
}
|
||||
}
|
||||
|
||||
// Abstract exhaustive test: all sizes up to 100,
|
||||
// all possible return values. If there are any small
|
||||
// corner cases, this test exercises them.
|
||||
func TestSearchExhaustive(t *testing.T) {
|
||||
for size := 0; size <= 100; size++ {
|
||||
for targ := 0; targ <= size; targ++ {
|
||||
i := Search(size, func(i int) bool { return i >= targ })
|
||||
if i != targ {
|
||||
t.Errorf("Search(%d, %d) = %d", size, targ, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,623 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:generate go run genzfunc.go
|
||||
|
||||
// Package sort provides primitives for sorting slices and user-defined
|
||||
// collections.
|
||||
//
|
||||
// This is a copy of the Go standard library's sort package with the
|
||||
// addition of some helpers for sorting slices and using func literals
|
||||
// to sort, rather than having to create a sorter type. See the
|
||||
// additional MakeInterface, SliceSorter, and Slice functions.
|
||||
// Discussion of moving such helpers into the standard library is
|
||||
// at:
|
||||
//
|
||||
// https://golang.org/issue/16721
|
||||
//
|
||||
// Per Go's "no +1 policy", please only leave a comment on that issue
|
||||
// if you have something unique to add. Use Github's emoji reactions
|
||||
// otherwise.
|
||||
package sort
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"go4.org/reflectutil"
|
||||
)
|
||||
|
||||
// A type, typically a collection, that satisfies sort.Interface can be
|
||||
// sorted by the routines in this package. The methods require that the
|
||||
// elements of the collection be enumerated by an integer index.
|
||||
type Interface interface {
|
||||
// Len is the number of elements in the collection.
|
||||
Len() int
|
||||
// Less reports whether the element with
|
||||
// index i should sort before the element with index j.
|
||||
Less(i, j int) bool
|
||||
// Swap swaps the elements with indexes i and j.
|
||||
Swap(i, j int)
|
||||
}
|
||||
|
||||
// lessSwap is a pair of Less and Swap function for use with the
|
||||
// auto-generated func-optimized variant of sort.go in
|
||||
// zfuncversion.go.
|
||||
type lessSwap struct {
|
||||
Less func(i, j int) bool
|
||||
Swap func(i, j int)
|
||||
}
|
||||
|
||||
// MakeInterface returns a sort Interface using the provided length
|
||||
// and pair of swap and less functions.
|
||||
func MakeInterface(length int, swap func(i, j int), less func(i, j int) bool) Interface {
|
||||
return &funcs{length, lessSwap{less, swap}}
|
||||
}
|
||||
|
||||
// SliceSorter returns a sort.Interface to sort the provided slice
|
||||
// using the provided less function.
|
||||
// If the provided interface is not a slice, the function panics.
|
||||
func SliceSorter(slice interface{}, less func(i, j int) bool) Interface {
|
||||
return MakeInterface(reflect.ValueOf(slice).Len(), reflectutil.Swapper(slice), less)
|
||||
}
|
||||
|
||||
// Slice sorts the provided slice using less.
|
||||
// If the provided interface is not a slice, the function panics.
|
||||
// The sort is not stable. For a stable sort, use sort.Stable with sort.SliceSorter.
|
||||
func Slice(slice interface{}, less func(i, j int) bool) {
|
||||
Sort(SliceSorter(slice, less))
|
||||
}
|
||||
|
||||
// funcs implements Interface, but is recognized by Sort and Stable
|
||||
// which use its lessSwap field with the non-interface sorting
|
||||
// routines in zfuncversion.go.
|
||||
type funcs struct {
|
||||
length int
|
||||
lessSwap
|
||||
}
|
||||
|
||||
func (f *funcs) Len() int { return f.length }
|
||||
func (f *funcs) Swap(i, j int) { f.lessSwap.Swap(i, j) }
|
||||
func (f *funcs) Less(i, j int) bool { return f.lessSwap.Less(i, j) }
|
||||
|
||||
// Insertion sort
|
||||
func insertionSort(data Interface, a, b int) {
|
||||
for i := a + 1; i < b; i++ {
|
||||
for j := i; j > a && data.Less(j, j-1); j-- {
|
||||
data.Swap(j, j-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// siftDown implements the heap property on data[lo, hi).
|
||||
// first is an offset into the array where the root of the heap lies.
|
||||
func siftDown(data Interface, lo, hi, first int) {
|
||||
root := lo
|
||||
for {
|
||||
child := 2*root + 1
|
||||
if child >= hi {
|
||||
break
|
||||
}
|
||||
if child+1 < hi && data.Less(first+child, first+child+1) {
|
||||
child++
|
||||
}
|
||||
if !data.Less(first+root, first+child) {
|
||||
return
|
||||
}
|
||||
data.Swap(first+root, first+child)
|
||||
root = child
|
||||
}
|
||||
}
|
||||
|
||||
func heapSort(data Interface, a, b int) {
|
||||
first := a
|
||||
lo := 0
|
||||
hi := b - a
|
||||
|
||||
// Build heap with greatest element at top.
|
||||
for i := (hi - 1) / 2; i >= 0; i-- {
|
||||
siftDown(data, i, hi, first)
|
||||
}
|
||||
|
||||
// Pop elements, largest first, into end of data.
|
||||
for i := hi - 1; i >= 0; i-- {
|
||||
data.Swap(first, first+i)
|
||||
siftDown(data, lo, i, first)
|
||||
}
|
||||
}
|
||||
|
||||
// Quicksort, loosely following Bentley and McIlroy,
|
||||
// ``Engineering a Sort Function,'' SP&E November 1993.
|
||||
|
||||
// medianOfThree moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
|
||||
func medianOfThree(data Interface, m1, m0, m2 int) {
|
||||
// sort 3 elements
|
||||
if data.Less(m1, m0) {
|
||||
data.Swap(m1, m0)
|
||||
}
|
||||
// data[m0] <= data[m1]
|
||||
if data.Less(m2, m1) {
|
||||
data.Swap(m2, m1)
|
||||
// data[m0] <= data[m2] && data[m1] < data[m2]
|
||||
if data.Less(m1, m0) {
|
||||
data.Swap(m1, m0)
|
||||
}
|
||||
}
|
||||
// now data[m0] <= data[m1] <= data[m2]
|
||||
}
|
||||
|
||||
func swapRange(data Interface, a, b, n int) {
|
||||
for i := 0; i < n; i++ {
|
||||
data.Swap(a+i, b+i)
|
||||
}
|
||||
}
|
||||
|
||||
func doPivot(data Interface, lo, hi int) (midlo, midhi int) {
|
||||
m := lo + (hi-lo)/2 // Written like this to avoid integer overflow.
|
||||
if hi-lo > 40 {
|
||||
// Tukey's ``Ninther,'' median of three medians of three.
|
||||
s := (hi - lo) / 8
|
||||
medianOfThree(data, lo, lo+s, lo+2*s)
|
||||
medianOfThree(data, m, m-s, m+s)
|
||||
medianOfThree(data, hi-1, hi-1-s, hi-1-2*s)
|
||||
}
|
||||
medianOfThree(data, lo, m, hi-1)
|
||||
|
||||
// Invariants are:
|
||||
// data[lo] = pivot (set up by ChoosePivot)
|
||||
// data[lo < i < a] < pivot
|
||||
// data[a <= i < b] <= pivot
|
||||
// data[b <= i < c] unexamined
|
||||
// data[c <= i < hi-1] > pivot
|
||||
// data[hi-1] >= pivot
|
||||
pivot := lo
|
||||
a, c := lo+1, hi-1
|
||||
|
||||
for ; a < c && data.Less(a, pivot); a++ {
|
||||
}
|
||||
b := a
|
||||
for {
|
||||
for ; b < c && !data.Less(pivot, b); b++ { // data[b] <= pivot
|
||||
}
|
||||
for ; b < c && data.Less(pivot, c-1); c-- { // data[c-1] > pivot
|
||||
}
|
||||
if b >= c {
|
||||
break
|
||||
}
|
||||
// data[b] > pivot; data[c-1] <= pivot
|
||||
data.Swap(b, c-1)
|
||||
b++
|
||||
c--
|
||||
}
|
||||
// If hi-c<3 then there are duplicates (by property of median of nine).
|
||||
// Let be a bit more conservative, and set border to 5.
|
||||
protect := hi-c < 5
|
||||
if !protect && hi-c < (hi-lo)/4 {
|
||||
// Lets test some points for equality to pivot
|
||||
dups := 0
|
||||
if !data.Less(pivot, hi-1) { // data[hi-1] = pivot
|
||||
data.Swap(c, hi-1)
|
||||
c++
|
||||
dups++
|
||||
}
|
||||
if !data.Less(b-1, pivot) { // data[b-1] = pivot
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// m-lo = (hi-lo)/2 > 6
|
||||
// b-lo > (hi-lo)*3/4-1 > 8
|
||||
// ==> m < b ==> data[m] <= pivot
|
||||
if !data.Less(m, pivot) { // data[m] = pivot
|
||||
data.Swap(m, b-1)
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// if at least 2 points are equal to pivot, assume skewed distribution
|
||||
protect = dups > 1
|
||||
}
|
||||
if protect {
|
||||
// Protect against a lot of duplicates
|
||||
// Add invariant:
|
||||
// data[a <= i < b] unexamined
|
||||
// data[b <= i < c] = pivot
|
||||
for {
|
||||
for ; a < b && !data.Less(b-1, pivot); b-- { // data[b] == pivot
|
||||
}
|
||||
for ; a < b && data.Less(a, pivot); a++ { // data[a] < pivot
|
||||
}
|
||||
if a >= b {
|
||||
break
|
||||
}
|
||||
// data[a] == pivot; data[b-1] < pivot
|
||||
data.Swap(a, b-1)
|
||||
a++
|
||||
b--
|
||||
}
|
||||
}
|
||||
// Swap pivot into middle
|
||||
data.Swap(pivot, b-1)
|
||||
return b - 1, c
|
||||
}
|
||||
|
||||
func quickSort(data Interface, a, b, maxDepth int) {
|
||||
for b-a > 12 { // Use ShellSort for slices <= 12 elements
|
||||
if maxDepth == 0 {
|
||||
heapSort(data, a, b)
|
||||
return
|
||||
}
|
||||
maxDepth--
|
||||
mlo, mhi := doPivot(data, a, b)
|
||||
// Avoiding recursion on the larger subproblem guarantees
|
||||
// a stack depth of at most lg(b-a).
|
||||
if mlo-a < b-mhi {
|
||||
quickSort(data, a, mlo, maxDepth)
|
||||
a = mhi // i.e., quickSort(data, mhi, b)
|
||||
} else {
|
||||
quickSort(data, mhi, b, maxDepth)
|
||||
b = mlo // i.e., quickSort(data, a, mlo)
|
||||
}
|
||||
}
|
||||
if b-a > 1 {
|
||||
// Do ShellSort pass with gap 6
|
||||
// It could be written in this simplified form cause b-a <= 12
|
||||
for i := a + 6; i < b; i++ {
|
||||
if data.Less(i, i-6) {
|
||||
data.Swap(i, i-6)
|
||||
}
|
||||
}
|
||||
insertionSort(data, a, b)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort sorts data.
|
||||
//
|
||||
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
|
||||
// data.Less and data.Swap. The sort is not guaranteed to be stable.
|
||||
//
|
||||
// To sort slices without creating a type, see Slice.
|
||||
func Sort(data Interface) {
|
||||
n := data.Len()
|
||||
if fs, ok := data.(*funcs); ok {
|
||||
quickSort_func(fs.lessSwap, 0, n, maxDepth(n))
|
||||
} else {
|
||||
quickSort(data, 0, n, maxDepth(n))
|
||||
}
|
||||
}
|
||||
|
||||
// With sorts data given the provided length, swap, and less
|
||||
// functions.
|
||||
// The sort is not guaranteed to be stable.
|
||||
func With(length int, swap func(i, j int), less func(i, j int) bool) {
|
||||
quickSort_func(lessSwap{less, swap}, 0, length, maxDepth(length))
|
||||
}
|
||||
|
||||
// maxDepth returns a threshold at which quicksort should switch
|
||||
// to heapsort. It returns 2*ceil(lg(n+1)).
|
||||
func maxDepth(n int) int {
|
||||
var depth int
|
||||
for i := n; i > 0; i >>= 1 {
|
||||
depth++
|
||||
}
|
||||
return depth * 2
|
||||
}
|
||||
|
||||
type reverse struct {
|
||||
// This embedded Interface permits Reverse to use the methods of
|
||||
// another Interface implementation.
|
||||
Interface
|
||||
}
|
||||
|
||||
// Less returns the opposite of the embedded implementation's Less method.
|
||||
func (r reverse) Less(i, j int) bool {
|
||||
return r.Interface.Less(j, i)
|
||||
}
|
||||
|
||||
// Reverse returns the reverse order for data.
|
||||
func Reverse(data Interface) Interface {
|
||||
return &reverse{data}
|
||||
}
|
||||
|
||||
// IsSorted reports whether data is sorted.
|
||||
func IsSorted(data Interface) bool {
|
||||
n := data.Len()
|
||||
for i := n - 1; i > 0; i-- {
|
||||
if data.Less(i, i-1) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Convenience types for common cases
|
||||
|
||||
// IntSlice attaches the methods of Interface to []int, sorting in increasing order.
|
||||
type IntSlice []int
|
||||
|
||||
func (p IntSlice) Len() int { return len(p) }
|
||||
func (p IntSlice) Less(i, j int) bool { return p[i] < p[j] }
|
||||
func (p IntSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
|
||||
// Sort is a convenience method.
|
||||
func (p IntSlice) Sort() { Sort(p) }
|
||||
|
||||
// Float64Slice attaches the methods of Interface to []float64, sorting in increasing order.
|
||||
type Float64Slice []float64
|
||||
|
||||
func (p Float64Slice) Len() int { return len(p) }
|
||||
func (p Float64Slice) Less(i, j int) bool { return p[i] < p[j] || isNaN(p[i]) && !isNaN(p[j]) }
|
||||
func (p Float64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
|
||||
// isNaN is a copy of math.IsNaN to avoid a dependency on the math package.
|
||||
func isNaN(f float64) bool {
|
||||
return f != f
|
||||
}
|
||||
|
||||
// Sort is a convenience method.
|
||||
func (p Float64Slice) Sort() { Sort(p) }
|
||||
|
||||
// StringSlice attaches the methods of Interface to []string, sorting in increasing order.
|
||||
type StringSlice []string
|
||||
|
||||
func (p StringSlice) Len() int { return len(p) }
|
||||
func (p StringSlice) Less(i, j int) bool { return p[i] < p[j] }
|
||||
func (p StringSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
|
||||
// Sort is a convenience method.
|
||||
func (p StringSlice) Sort() { Sort(p) }
|
||||
|
||||
// Convenience wrappers for common cases
|
||||
|
||||
// Ints sorts a slice of ints in increasing order.
|
||||
func Ints(a []int) { Sort(IntSlice(a)) }
|
||||
|
||||
// Float64s sorts a slice of float64s in increasing order.
|
||||
func Float64s(a []float64) { Sort(Float64Slice(a)) }
|
||||
|
||||
// Strings sorts a slice of strings in increasing order.
|
||||
func Strings(a []string) { Sort(StringSlice(a)) }
|
||||
|
||||
// IntsAreSorted tests whether a slice of ints is sorted in increasing order.
|
||||
func IntsAreSorted(a []int) bool { return IsSorted(IntSlice(a)) }
|
||||
|
||||
// Float64sAreSorted tests whether a slice of float64s is sorted in increasing order.
|
||||
func Float64sAreSorted(a []float64) bool { return IsSorted(Float64Slice(a)) }
|
||||
|
||||
// StringsAreSorted tests whether a slice of strings is sorted in increasing order.
|
||||
func StringsAreSorted(a []string) bool { return IsSorted(StringSlice(a)) }
|
||||
|
||||
// Notes on stable sorting:
|
||||
// The used algorithms are simple and provable correct on all input and use
|
||||
// only logarithmic additional stack space. They perform well if compared
|
||||
// experimentally to other stable in-place sorting algorithms.
|
||||
//
|
||||
// Remarks on other algorithms evaluated:
|
||||
// - GCC's 4.6.3 stable_sort with merge_without_buffer from libstdc++:
|
||||
// Not faster.
|
||||
// - GCC's __rotate for block rotations: Not faster.
|
||||
// - "Practical in-place mergesort" from Jyrki Katajainen, Tomi A. Pasanen
|
||||
// and Jukka Teuhola; Nordic Journal of Computing 3,1 (1996), 27-40:
|
||||
// The given algorithms are in-place, number of Swap and Assignments
|
||||
// grow as n log n but the algorithm is not stable.
|
||||
// - "Fast Stable In-Place Sorting with O(n) Data Moves" J.I. Munro and
|
||||
// V. Raman in Algorithmica (1996) 16, 115-160:
|
||||
// This algorithm either needs additional 2n bits or works only if there
|
||||
// are enough different elements available to encode some permutations
|
||||
// which have to be undone later (so not stable on any input).
|
||||
// - All the optimal in-place sorting/merging algorithms I found are either
|
||||
// unstable or rely on enough different elements in each step to encode the
|
||||
// performed block rearrangements. See also "In-Place Merging Algorithms",
|
||||
// Denham Coates-Evely, Department of Computer Science, Kings College,
|
||||
// January 2004 and the references in there.
|
||||
// - Often "optimal" algorithms are optimal in the number of assignments
|
||||
// but Interface has only Swap as operation.
|
||||
|
||||
// Stable sorts data while keeping the original order of equal elements.
|
||||
//
|
||||
// It makes one call to data.Len to determine n, O(n*log(n)) calls to
|
||||
// data.Less and O(n*log(n)*log(n)) calls to data.Swap.
|
||||
func Stable(data Interface) {
|
||||
if fs, ok := data.(*funcs); ok {
|
||||
stable_func(fs.lessSwap, fs.length)
|
||||
} else {
|
||||
stable(data, data.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func stable(data Interface, n int) {
|
||||
blockSize := 20 // must be > 0
|
||||
a, b := 0, blockSize
|
||||
for b <= n {
|
||||
insertionSort(data, a, b)
|
||||
a = b
|
||||
b += blockSize
|
||||
}
|
||||
insertionSort(data, a, n)
|
||||
|
||||
for blockSize < n {
|
||||
a, b = 0, 2*blockSize
|
||||
for b <= n {
|
||||
symMerge(data, a, a+blockSize, b)
|
||||
a = b
|
||||
b += 2 * blockSize
|
||||
}
|
||||
if m := a + blockSize; m < n {
|
||||
symMerge(data, a, m, n)
|
||||
}
|
||||
blockSize *= 2
|
||||
}
|
||||
}
|
||||
|
||||
// SymMerge merges the two sorted subsequences data[a:m] and data[m:b] using
|
||||
// the SymMerge algorithm from Pok-Son Kim and Arne Kutzner, "Stable Minimum
|
||||
// Storage Merging by Symmetric Comparisons", in Susanne Albers and Tomasz
|
||||
// Radzik, editors, Algorithms - ESA 2004, volume 3221 of Lecture Notes in
|
||||
// Computer Science, pages 714-723. Springer, 2004.
|
||||
//
|
||||
// Let M = m-a and N = b-n. Wolog M < N.
|
||||
// The recursion depth is bound by ceil(log(N+M)).
|
||||
// The algorithm needs O(M*log(N/M + 1)) calls to data.Less.
|
||||
// The algorithm needs O((M+N)*log(M)) calls to data.Swap.
|
||||
//
|
||||
// The paper gives O((M+N)*log(M)) as the number of assignments assuming a
|
||||
// rotation algorithm which uses O(M+N+gcd(M+N)) assignments. The argumentation
|
||||
// in the paper carries through for Swap operations, especially as the block
|
||||
// swapping rotate uses only O(M+N) Swaps.
|
||||
//
|
||||
// symMerge assumes non-degenerate arguments: a < m && m < b.
|
||||
// Having the caller check this condition eliminates many leaf recursion calls,
|
||||
// which improves performance.
|
||||
func symMerge(data Interface, a, m, b int) {
|
||||
// Avoid unnecessary recursions of symMerge
|
||||
// by direct insertion of data[a] into data[m:b]
|
||||
// if data[a:m] only contains one element.
|
||||
if m-a == 1 {
|
||||
// Use binary search to find the lowest index i
|
||||
// such that data[i] >= data[a] for m <= i < b.
|
||||
// Exit the search loop with i == b in case no such index exists.
|
||||
i := m
|
||||
j := b
|
||||
for i < j {
|
||||
h := i + (j-i)/2
|
||||
if data.Less(h, a) {
|
||||
i = h + 1
|
||||
} else {
|
||||
j = h
|
||||
}
|
||||
}
|
||||
// Swap values until data[a] reaches the position before i.
|
||||
for k := a; k < i-1; k++ {
|
||||
data.Swap(k, k+1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Avoid unnecessary recursions of symMerge
|
||||
// by direct insertion of data[m] into data[a:m]
|
||||
// if data[m:b] only contains one element.
|
||||
if b-m == 1 {
|
||||
// Use binary search to find the lowest index i
|
||||
// such that data[i] > data[m] for a <= i < m.
|
||||
// Exit the search loop with i == m in case no such index exists.
|
||||
i := a
|
||||
j := m
|
||||
for i < j {
|
||||
h := i + (j-i)/2
|
||||
if !data.Less(m, h) {
|
||||
i = h + 1
|
||||
} else {
|
||||
j = h
|
||||
}
|
||||
}
|
||||
// Swap values until data[m] reaches the position i.
|
||||
for k := m; k > i; k-- {
|
||||
data.Swap(k, k-1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
mid := a + (b-a)/2
|
||||
n := mid + m
|
||||
var start, r int
|
||||
if m > mid {
|
||||
start = n - b
|
||||
r = mid
|
||||
} else {
|
||||
start = a
|
||||
r = m
|
||||
}
|
||||
p := n - 1
|
||||
|
||||
for start < r {
|
||||
c := start + (r-start)/2
|
||||
if !data.Less(p-c, c) {
|
||||
start = c + 1
|
||||
} else {
|
||||
r = c
|
||||
}
|
||||
}
|
||||
|
||||
end := n - start
|
||||
if start < m && m < end {
|
||||
rotate(data, start, m, end)
|
||||
}
|
||||
if a < start && start < mid {
|
||||
symMerge(data, a, start, mid)
|
||||
}
|
||||
if mid < end && end < b {
|
||||
symMerge(data, mid, end, b)
|
||||
}
|
||||
}
|
||||
|
||||
// Rotate two consecutives blocks u = data[a:m] and v = data[m:b] in data:
|
||||
// Data of the form 'x u v y' is changed to 'x v u y'.
|
||||
// Rotate performs at most b-a many calls to data.Swap.
|
||||
// Rotate assumes non-degenerate arguments: a < m && m < b.
|
||||
func rotate(data Interface, a, m, b int) {
|
||||
i := m - a
|
||||
j := b - m
|
||||
|
||||
for i != j {
|
||||
if i > j {
|
||||
swapRange(data, m-i, m, j)
|
||||
i -= j
|
||||
} else {
|
||||
swapRange(data, m-i, m+j-i, i)
|
||||
j -= i
|
||||
}
|
||||
}
|
||||
// i == j
|
||||
swapRange(data, m-i, m, i)
|
||||
}
|
||||
|
||||
/*
|
||||
Complexity of Stable Sorting
|
||||
|
||||
|
||||
Complexity of block swapping rotation
|
||||
|
||||
Each Swap puts one new element into its correct, final position.
|
||||
Elements which reach their final position are no longer moved.
|
||||
Thus block swapping rotation needs |u|+|v| calls to Swaps.
|
||||
This is best possible as each element might need a move.
|
||||
|
||||
Pay attention when comparing to other optimal algorithms which
|
||||
typically count the number of assignments instead of swaps:
|
||||
E.g. the optimal algorithm of Dudzinski and Dydek for in-place
|
||||
rotations uses O(u + v + gcd(u,v)) assignments which is
|
||||
better than our O(3 * (u+v)) as gcd(u,v) <= u.
|
||||
|
||||
|
||||
Stable sorting by SymMerge and BlockSwap rotations
|
||||
|
||||
SymMerg complexity for same size input M = N:
|
||||
Calls to Less: O(M*log(N/M+1)) = O(N*log(2)) = O(N)
|
||||
Calls to Swap: O((M+N)*log(M)) = O(2*N*log(N)) = O(N*log(N))
|
||||
|
||||
(The following argument does not fuzz over a missing -1 or
|
||||
other stuff which does not impact the final result).
|
||||
|
||||
Let n = data.Len(). Assume n = 2^k.
|
||||
|
||||
Plain merge sort performs log(n) = k iterations.
|
||||
On iteration i the algorithm merges 2^(k-i) blocks, each of size 2^i.
|
||||
|
||||
Thus iteration i of merge sort performs:
|
||||
Calls to Less O(2^(k-i) * 2^i) = O(2^k) = O(2^log(n)) = O(n)
|
||||
Calls to Swap O(2^(k-i) * 2^i * log(2^i)) = O(2^k * i) = O(n*i)
|
||||
|
||||
In total k = log(n) iterations are performed; so in total:
|
||||
Calls to Less O(log(n) * n)
|
||||
Calls to Swap O(n + 2*n + 3*n + ... + (k-1)*n + k*n)
|
||||
= O((k/2) * k * n) = O(n * k^2) = O(n * log^2(n))
|
||||
|
||||
|
||||
Above results should generalize to arbitrary n = 2^k + p
|
||||
and should not be influenced by the initial insertion sort phase:
|
||||
Insertion sort is O(n^2) on Swap and Less, thus O(bs^2) per block of
|
||||
size bs at n/bs blocks: O(bs*n) Swaps and Less during insertion sort.
|
||||
Merge sort iterations start at i = log(bs). With t = log(bs) constant:
|
||||
Calls to Less O((log(n)-t) * n + bs*n) = O(log(n)*n + (bs-t)*n)
|
||||
= O(n * log(n))
|
||||
Calls to Swap O(n * log^2(n) - (t^2+t)/2*n) = O(n * log^2(n))
|
||||
|
||||
*/
|
||||
@ -1,695 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"go4.org/reflectutil"
|
||||
. "go4.org/sort"
|
||||
)
|
||||
|
||||
var ints = [...]int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586}
|
||||
var float64s = [...]float64{74.3, 59.0, math.Inf(1), 238.2, -784.0, 2.3, math.NaN(), math.NaN(), math.Inf(-1), 9845.768, -959.7485, 905, 7.8, 7.8}
|
||||
var strings = [...]string{"", "Hello", "foo", "bar", "foo", "f00", "%*&^*&^&", "***"}
|
||||
|
||||
func TestSlice(t *testing.T) {
|
||||
s := []int{5, 4, 3, 2, 1}
|
||||
want := []int{1, 2, 3, 4, 5}
|
||||
Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
||||
if !reflect.DeepEqual(s, want) {
|
||||
t.Errorf("sorted = %v; want %v", s, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortIntSlice(t *testing.T) {
|
||||
data := ints
|
||||
a := IntSlice(data[0:])
|
||||
Sort(a)
|
||||
if !IsSorted(a) {
|
||||
t.Errorf("sorted %v", ints)
|
||||
t.Errorf(" got %v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortFloat64Slice(t *testing.T) {
|
||||
data := float64s
|
||||
a := Float64Slice(data[0:])
|
||||
Sort(a)
|
||||
if !IsSorted(a) {
|
||||
t.Errorf("sorted %v", float64s)
|
||||
t.Errorf(" got %v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortStringSlice(t *testing.T) {
|
||||
data := strings
|
||||
a := StringSlice(data[0:])
|
||||
Sort(a)
|
||||
if !IsSorted(a) {
|
||||
t.Errorf("sorted %v", strings)
|
||||
t.Errorf(" got %v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInts(t *testing.T) {
|
||||
data := ints
|
||||
Ints(data[0:])
|
||||
if !IntsAreSorted(data[0:]) {
|
||||
t.Errorf("sorted %v", ints)
|
||||
t.Errorf(" got %v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFloat64s(t *testing.T) {
|
||||
data := float64s
|
||||
Float64s(data[0:])
|
||||
if !Float64sAreSorted(data[0:]) {
|
||||
t.Errorf("sorted %v", float64s)
|
||||
t.Errorf(" got %v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrings(t *testing.T) {
|
||||
data := strings
|
||||
Strings(data[0:])
|
||||
if !StringsAreSorted(data[0:]) {
|
||||
t.Errorf("sorted %v", strings)
|
||||
t.Errorf(" got %v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStringsWithSwapper(t *testing.T) {
|
||||
data := strings
|
||||
With(len(data), reflectutil.Swapper(data[:]), func(i, j int) bool {
|
||||
return data[i] < data[j]
|
||||
})
|
||||
if !StringsAreSorted(data[:]) {
|
||||
t.Errorf("sorted %v", strings)
|
||||
t.Errorf(" got %v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortLarge_Random(t *testing.T) {
|
||||
n := 1000000
|
||||
if testing.Short() {
|
||||
n /= 100
|
||||
}
|
||||
data := make([]int, n)
|
||||
for i := 0; i < len(data); i++ {
|
||||
data[i] = rand.Intn(100)
|
||||
}
|
||||
if IntsAreSorted(data) {
|
||||
t.Fatalf("terrible rand.rand")
|
||||
}
|
||||
Ints(data)
|
||||
if !IntsAreSorted(data) {
|
||||
t.Errorf("sort didn't sort - 1M ints")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReverseSortIntSlice(t *testing.T) {
|
||||
data := ints
|
||||
data1 := ints
|
||||
a := IntSlice(data[0:])
|
||||
Sort(a)
|
||||
r := IntSlice(data1[0:])
|
||||
Sort(Reverse(r))
|
||||
for i := 0; i < len(data); i++ {
|
||||
if a[i] != r[len(data)-1-i] {
|
||||
t.Errorf("reverse sort didn't sort")
|
||||
}
|
||||
if i > len(data)/2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type nonDeterministicTestingData struct {
|
||||
r *rand.Rand
|
||||
}
|
||||
|
||||
func (t *nonDeterministicTestingData) Len() int {
|
||||
return 500
|
||||
}
|
||||
func (t *nonDeterministicTestingData) Less(i, j int) bool {
|
||||
if i < 0 || j < 0 || i >= t.Len() || j >= t.Len() {
|
||||
panic("nondeterministic comparison out of bounds")
|
||||
}
|
||||
return t.r.Float32() < 0.5
|
||||
}
|
||||
func (t *nonDeterministicTestingData) Swap(i, j int) {
|
||||
if i < 0 || j < 0 || i >= t.Len() || j >= t.Len() {
|
||||
panic("nondeterministic comparison out of bounds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonDeterministicComparison(t *testing.T) {
|
||||
// Ensure that sort.Sort does not panic when Less returns inconsistent results.
|
||||
// See https://golang.org/issue/14377.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Error(r)
|
||||
}
|
||||
}()
|
||||
|
||||
td := &nonDeterministicTestingData{
|
||||
r: rand.New(rand.NewSource(0)),
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
Sort(td)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSortString1K(b *testing.B) {
|
||||
b.StopTimer()
|
||||
unsorted := make([]string, 1<<10)
|
||||
for i := range unsorted {
|
||||
unsorted[i] = strconv.Itoa(i ^ 0x2cc)
|
||||
}
|
||||
data := make([]string, len(unsorted))
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(data, unsorted)
|
||||
b.StartTimer()
|
||||
Strings(data)
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSortString1K_With(b *testing.B) {
|
||||
b.StopTimer()
|
||||
unsorted := make([]string, 1<<10)
|
||||
for i := range unsorted {
|
||||
unsorted[i] = strconv.Itoa(i ^ 0x2cc)
|
||||
}
|
||||
data := make([]string, len(unsorted))
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(data, unsorted)
|
||||
b.StartTimer()
|
||||
With(len(data),
|
||||
func(i, j int) { data[i], data[j] = data[j], data[i] },
|
||||
func(i, j int) bool {
|
||||
return data[i] < data[j]
|
||||
})
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSortString1K_WithSwapper(b *testing.B) {
|
||||
b.StopTimer()
|
||||
unsorted := make([]string, 1<<10)
|
||||
for i := range unsorted {
|
||||
unsorted[i] = strconv.Itoa(i ^ 0x2cc)
|
||||
}
|
||||
data := make([]string, len(unsorted))
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(data, unsorted)
|
||||
b.StartTimer()
|
||||
With(len(data), reflectutil.Swapper(data), func(i, j int) bool {
|
||||
return data[i] < data[j]
|
||||
})
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStableString1K(b *testing.B) {
|
||||
b.StopTimer()
|
||||
unsorted := make([]string, 1<<10)
|
||||
for i := 0; i < len(data); i++ {
|
||||
unsorted[i] = strconv.Itoa(i ^ 0x2cc)
|
||||
}
|
||||
data := make([]string, len(unsorted))
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(data, unsorted)
|
||||
b.StartTimer()
|
||||
Stable(StringSlice(data))
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSortInt1K(b *testing.B) {
|
||||
b.StopTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
data := make([]int, 1<<10)
|
||||
for i := 0; i < len(data); i++ {
|
||||
data[i] = i ^ 0x2cc
|
||||
}
|
||||
b.StartTimer()
|
||||
Ints(data)
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStableInt1K(b *testing.B) {
|
||||
b.StopTimer()
|
||||
unsorted := make([]int, 1<<10)
|
||||
for i := range unsorted {
|
||||
unsorted[i] = i ^ 0x2cc
|
||||
}
|
||||
data := make([]int, len(unsorted))
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(data, unsorted)
|
||||
b.StartTimer()
|
||||
Stable(IntSlice(data))
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStableInt1K_With(b *testing.B) {
|
||||
b.StopTimer()
|
||||
unsorted := make([]int, 1<<10)
|
||||
for i := range unsorted {
|
||||
unsorted[i] = i ^ 0x2cc
|
||||
}
|
||||
data := make([]int, len(unsorted))
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(data, unsorted)
|
||||
b.StartTimer()
|
||||
Stable(MakeInterface(
|
||||
len(data),
|
||||
func(i, j int) { data[i], data[j] = data[j], data[i] },
|
||||
func(i, j int) bool { return data[i] < data[j] },
|
||||
))
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStableInt1K_WithSwapper(b *testing.B) {
|
||||
b.StopTimer()
|
||||
unsorted := make([]int, 1<<10)
|
||||
for i := range unsorted {
|
||||
unsorted[i] = i ^ 0x2cc
|
||||
}
|
||||
data := make([]int, len(unsorted))
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(data, unsorted)
|
||||
b.StartTimer()
|
||||
Stable(MakeInterface(len(data), reflectutil.Swapper(data), func(i, j int) bool {
|
||||
return data[i] < data[j]
|
||||
}))
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSortInt64K(b *testing.B) {
|
||||
b.StopTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
data := make([]int, 1<<16)
|
||||
for i := 0; i < len(data); i++ {
|
||||
data[i] = i ^ 0xcccc
|
||||
}
|
||||
b.StartTimer()
|
||||
Ints(data)
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStableInt64K(b *testing.B) {
|
||||
b.StopTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
data := make([]int, 1<<16)
|
||||
for i := 0; i < len(data); i++ {
|
||||
data[i] = i ^ 0xcccc
|
||||
}
|
||||
b.StartTimer()
|
||||
Stable(IntSlice(data))
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
_Sawtooth = iota
|
||||
_Rand
|
||||
_Stagger
|
||||
_Plateau
|
||||
_Shuffle
|
||||
_NDist
|
||||
)
|
||||
|
||||
const (
|
||||
_Copy = iota
|
||||
_Reverse
|
||||
_ReverseFirstHalf
|
||||
_ReverseSecondHalf
|
||||
_Sorted
|
||||
_Dither
|
||||
_NMode
|
||||
)
|
||||
|
||||
type testingData struct {
|
||||
desc string
|
||||
t *testing.T
|
||||
data []int
|
||||
maxswap int // number of swaps allowed
|
||||
ncmp, nswap int
|
||||
}
|
||||
|
||||
func (d *testingData) Len() int { return len(d.data) }
|
||||
func (d *testingData) Less(i, j int) bool {
|
||||
d.ncmp++
|
||||
return d.data[i] < d.data[j]
|
||||
}
|
||||
func (d *testingData) Swap(i, j int) {
|
||||
if d.nswap >= d.maxswap {
|
||||
d.t.Errorf("%s: used %d swaps sorting slice of %d", d.desc, d.nswap, len(d.data))
|
||||
d.t.FailNow()
|
||||
}
|
||||
d.nswap++
|
||||
d.data[i], d.data[j] = d.data[j], d.data[i]
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func lg(n int) int {
|
||||
i := 0
|
||||
for 1<<uint(i) < n {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func testBentleyMcIlroy(t *testing.T, sort func(Interface), maxswap func(int) int) {
|
||||
sizes := []int{100, 1023, 1024, 1025}
|
||||
if testing.Short() {
|
||||
sizes = []int{100, 127, 128, 129}
|
||||
}
|
||||
dists := []string{"sawtooth", "rand", "stagger", "plateau", "shuffle"}
|
||||
modes := []string{"copy", "reverse", "reverse1", "reverse2", "sort", "dither"}
|
||||
var tmp1, tmp2 [1025]int
|
||||
for _, n := range sizes {
|
||||
for m := 1; m < 2*n; m *= 2 {
|
||||
for dist := 0; dist < _NDist; dist++ {
|
||||
j := 0
|
||||
k := 1
|
||||
data := tmp1[0:n]
|
||||
for i := 0; i < n; i++ {
|
||||
switch dist {
|
||||
case _Sawtooth:
|
||||
data[i] = i % m
|
||||
case _Rand:
|
||||
data[i] = rand.Intn(m)
|
||||
case _Stagger:
|
||||
data[i] = (i*m + i) % n
|
||||
case _Plateau:
|
||||
data[i] = min(i, m)
|
||||
case _Shuffle:
|
||||
if rand.Intn(m) != 0 {
|
||||
j += 2
|
||||
data[i] = j
|
||||
} else {
|
||||
k += 2
|
||||
data[i] = k
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mdata := tmp2[0:n]
|
||||
for mode := 0; mode < _NMode; mode++ {
|
||||
switch mode {
|
||||
case _Copy:
|
||||
for i := 0; i < n; i++ {
|
||||
mdata[i] = data[i]
|
||||
}
|
||||
case _Reverse:
|
||||
for i := 0; i < n; i++ {
|
||||
mdata[i] = data[n-i-1]
|
||||
}
|
||||
case _ReverseFirstHalf:
|
||||
for i := 0; i < n/2; i++ {
|
||||
mdata[i] = data[n/2-i-1]
|
||||
}
|
||||
for i := n / 2; i < n; i++ {
|
||||
mdata[i] = data[i]
|
||||
}
|
||||
case _ReverseSecondHalf:
|
||||
for i := 0; i < n/2; i++ {
|
||||
mdata[i] = data[i]
|
||||
}
|
||||
for i := n / 2; i < n; i++ {
|
||||
mdata[i] = data[n-(i-n/2)-1]
|
||||
}
|
||||
case _Sorted:
|
||||
for i := 0; i < n; i++ {
|
||||
mdata[i] = data[i]
|
||||
}
|
||||
// Ints is known to be correct
|
||||
// because mode Sort runs after mode _Copy.
|
||||
Ints(mdata)
|
||||
case _Dither:
|
||||
for i := 0; i < n; i++ {
|
||||
mdata[i] = data[i] + i%5
|
||||
}
|
||||
}
|
||||
|
||||
desc := fmt.Sprintf("n=%d m=%d dist=%s mode=%s", n, m, dists[dist], modes[mode])
|
||||
d := &testingData{desc: desc, t: t, data: mdata[0:n], maxswap: maxswap(n)}
|
||||
sort(d)
|
||||
// Uncomment if you are trying to improve the number of compares/swaps.
|
||||
//t.Logf("%s: ncmp=%d, nswp=%d", desc, d.ncmp, d.nswap)
|
||||
|
||||
// If we were testing C qsort, we'd have to make a copy
|
||||
// of the slice and sort it ourselves and then compare
|
||||
// x against it, to ensure that qsort was only permuting
|
||||
// the data, not (for example) overwriting it with zeros.
|
||||
//
|
||||
// In go, we don't have to be so paranoid: since the only
|
||||
// mutating method Sort can call is TestingData.swap,
|
||||
// it suffices here just to check that the final slice is sorted.
|
||||
if !IntsAreSorted(mdata) {
|
||||
t.Errorf("%s: ints not sorted", desc)
|
||||
t.Errorf("\t%v", mdata)
|
||||
t.FailNow()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortBM(t *testing.T) {
|
||||
testBentleyMcIlroy(t, Sort, func(n int) int { return n * lg(n) * 12 / 10 })
|
||||
}
|
||||
|
||||
func TestHeapsortBM(t *testing.T) {
|
||||
testBentleyMcIlroy(t, Heapsort, func(n int) int { return n * lg(n) * 12 / 10 })
|
||||
}
|
||||
|
||||
func TestStableBM(t *testing.T) {
|
||||
testBentleyMcIlroy(t, Stable, func(n int) int { return n * lg(n) * lg(n) / 3 })
|
||||
}
|
||||
|
||||
// This is based on the "antiquicksort" implementation by M. Douglas McIlroy.
|
||||
// See http://www.cs.dartmouth.edu/~doug/mdmspe.pdf for more info.
|
||||
type adversaryTestingData struct {
|
||||
data []int
|
||||
keys map[int]int
|
||||
candidate int
|
||||
}
|
||||
|
||||
func (d *adversaryTestingData) Len() int { return len(d.data) }
|
||||
|
||||
func (d *adversaryTestingData) Less(i, j int) bool {
|
||||
if _, present := d.keys[i]; !present {
|
||||
if _, present := d.keys[j]; !present {
|
||||
if i == d.candidate {
|
||||
d.keys[i] = len(d.keys)
|
||||
} else {
|
||||
d.keys[j] = len(d.keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, present := d.keys[i]; !present {
|
||||
d.candidate = i
|
||||
return false
|
||||
}
|
||||
if _, present := d.keys[j]; !present {
|
||||
d.candidate = j
|
||||
return true
|
||||
}
|
||||
|
||||
return d.keys[i] >= d.keys[j]
|
||||
}
|
||||
|
||||
func (d *adversaryTestingData) Swap(i, j int) {
|
||||
d.data[i], d.data[j] = d.data[j], d.data[i]
|
||||
}
|
||||
|
||||
func TestAdversary(t *testing.T) {
|
||||
const size = 100
|
||||
data := make([]int, size)
|
||||
for i := 0; i < size; i++ {
|
||||
data[i] = i
|
||||
}
|
||||
|
||||
d := &adversaryTestingData{data, make(map[int]int), 0}
|
||||
Sort(d) // This should degenerate to heapsort.
|
||||
}
|
||||
|
||||
func TestStableInts(t *testing.T) {
|
||||
data := ints
|
||||
Stable(IntSlice(data[0:]))
|
||||
if !IntsAreSorted(data[0:]) {
|
||||
t.Errorf("nsorted %v\n got %v", ints, data)
|
||||
}
|
||||
}
|
||||
|
||||
type intPairs []struct {
|
||||
a, b int
|
||||
}
|
||||
|
||||
// IntPairs compare on a only.
|
||||
func (d intPairs) Len() int { return len(d) }
|
||||
func (d intPairs) Less(i, j int) bool { return d[i].a < d[j].a }
|
||||
func (d intPairs) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
||||
|
||||
// Record initial order in B.
|
||||
func (d intPairs) initB() {
|
||||
for i := range d {
|
||||
d[i].b = i
|
||||
}
|
||||
}
|
||||
|
||||
// InOrder checks if a-equal elements were not reordered.
|
||||
func (d intPairs) inOrder() bool {
|
||||
lastA, lastB := -1, 0
|
||||
for i := 0; i < len(d); i++ {
|
||||
if lastA != d[i].a {
|
||||
lastA = d[i].a
|
||||
lastB = d[i].b
|
||||
continue
|
||||
}
|
||||
if d[i].b <= lastB {
|
||||
return false
|
||||
}
|
||||
lastB = d[i].b
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestStability(t *testing.T) {
|
||||
n, m := 100000, 1000
|
||||
if testing.Short() {
|
||||
n, m = 1000, 100
|
||||
}
|
||||
data := make(intPairs, n)
|
||||
|
||||
// random distribution
|
||||
for i := 0; i < len(data); i++ {
|
||||
data[i].a = rand.Intn(m)
|
||||
}
|
||||
if IsSorted(data) {
|
||||
t.Fatalf("terrible rand.rand")
|
||||
}
|
||||
data.initB()
|
||||
Stable(data)
|
||||
if !IsSorted(data) {
|
||||
t.Errorf("Stable didn't sort %d ints", n)
|
||||
}
|
||||
if !data.inOrder() {
|
||||
t.Errorf("Stable wasn't stable on %d ints", n)
|
||||
}
|
||||
|
||||
// already sorted
|
||||
data.initB()
|
||||
Stable(data)
|
||||
if !IsSorted(data) {
|
||||
t.Errorf("Stable shuffled sorted %d ints (order)", n)
|
||||
}
|
||||
if !data.inOrder() {
|
||||
t.Errorf("Stable shuffled sorted %d ints (stability)", n)
|
||||
}
|
||||
|
||||
// sorted reversed
|
||||
for i := 0; i < len(data); i++ {
|
||||
data[i].a = len(data) - i
|
||||
}
|
||||
data.initB()
|
||||
Stable(data)
|
||||
if !IsSorted(data) {
|
||||
t.Errorf("Stable didn't sort %d ints", n)
|
||||
}
|
||||
if !data.inOrder() {
|
||||
t.Errorf("Stable wasn't stable on %d ints", n)
|
||||
}
|
||||
}
|
||||
|
||||
var countOpsSizes = []int{1e2, 3e2, 1e3, 3e3, 1e4, 3e4, 1e5, 3e5, 1e6}
|
||||
|
||||
func countOps(t *testing.T, algo func(Interface), name string) {
|
||||
sizes := countOpsSizes
|
||||
if testing.Short() {
|
||||
sizes = sizes[:5]
|
||||
}
|
||||
if !testing.Verbose() {
|
||||
t.Skip("Counting skipped as non-verbose mode.")
|
||||
}
|
||||
for _, n := range sizes {
|
||||
td := testingData{
|
||||
desc: name,
|
||||
t: t,
|
||||
data: make([]int, n),
|
||||
maxswap: 1<<31 - 1,
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
td.data[i] = rand.Intn(n / 5)
|
||||
}
|
||||
algo(&td)
|
||||
t.Logf("%s %8d elements: %11d Swap, %10d Less", name, n, td.nswap, td.ncmp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountStableOps(t *testing.T) { countOps(t, Stable, "Stable") }
|
||||
func TestCountSortOps(t *testing.T) { countOps(t, Sort, "Sort ") }
|
||||
|
||||
func bench(b *testing.B, size int, algo func(Interface), name string) {
|
||||
b.StopTimer()
|
||||
data := make(intPairs, size)
|
||||
x := ^uint32(0)
|
||||
for i := 0; i < b.N; i++ {
|
||||
for n := size - 3; n <= size+3; n++ {
|
||||
for i := 0; i < len(data); i++ {
|
||||
x += x
|
||||
x ^= 1
|
||||
if int32(x) < 0 {
|
||||
x ^= 0x88888eef
|
||||
}
|
||||
data[i].a = int(x % uint32(n/5))
|
||||
}
|
||||
data.initB()
|
||||
b.StartTimer()
|
||||
algo(data)
|
||||
b.StopTimer()
|
||||
if !IsSorted(data) {
|
||||
b.Errorf("%s did not sort %d ints", name, n)
|
||||
}
|
||||
if name == "Stable" && !data.inOrder() {
|
||||
b.Errorf("%s unstable on %d ints", name, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSort1e2(b *testing.B) { bench(b, 1e2, Sort, "Sort") }
|
||||
func BenchmarkStable1e2(b *testing.B) { bench(b, 1e2, Stable, "Stable") }
|
||||
func BenchmarkSort1e4(b *testing.B) { bench(b, 1e4, Sort, "Sort") }
|
||||
func BenchmarkStable1e4(b *testing.B) { bench(b, 1e4, Stable, "Stable") }
|
||||
func BenchmarkSort1e6(b *testing.B) { bench(b, 1e6, Sort, "Sort") }
|
||||
func BenchmarkStable1e6(b *testing.B) { bench(b, 1e6, Stable, "Stable") }
|
||||
@ -1,265 +0,0 @@
|
||||
// DO NOT EDIT; AUTO-GENERATED from sort.go using genzfunc.go
|
||||
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package sort
|
||||
|
||||
// Auto-generated variant of sort.go:insertionSort
|
||||
func insertionSort_func(data lessSwap, a, b int) {
|
||||
for i := a + 1; i < b; i++ {
|
||||
for j := i; j > a && data.Less(j, j-1); j-- {
|
||||
data.Swap(j, j-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:siftDown
|
||||
func siftDown_func(data lessSwap, lo, hi, first int) {
|
||||
root := lo
|
||||
for {
|
||||
child := 2*root + 1
|
||||
if child >= hi {
|
||||
break
|
||||
}
|
||||
if child+1 < hi && data.Less(first+child, first+child+1) {
|
||||
child++
|
||||
}
|
||||
if !data.Less(first+root, first+child) {
|
||||
return
|
||||
}
|
||||
data.Swap(first+root, first+child)
|
||||
root = child
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:heapSort
|
||||
func heapSort_func(data lessSwap, a, b int) {
|
||||
first := a
|
||||
lo := 0
|
||||
hi := b - a
|
||||
for i := (hi - 1) / 2; i >= 0; i-- {
|
||||
siftDown_func(data, i, hi, first)
|
||||
}
|
||||
for i := hi - 1; i >= 0; i-- {
|
||||
data.Swap(first, first+i)
|
||||
siftDown_func(data, lo, i, first)
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:medianOfThree
|
||||
func medianOfThree_func(data lessSwap, m1, m0, m2 int) {
|
||||
if data.Less(m1, m0) {
|
||||
data.Swap(m1, m0)
|
||||
}
|
||||
if data.Less(m2, m1) {
|
||||
data.Swap(m2, m1)
|
||||
if data.Less(m1, m0) {
|
||||
data.Swap(m1, m0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:swapRange
|
||||
func swapRange_func(data lessSwap, a, b, n int) {
|
||||
for i := 0; i < n; i++ {
|
||||
data.Swap(a+i, b+i)
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:doPivot
|
||||
func doPivot_func(data lessSwap, lo, hi int) (midlo, midhi int) {
|
||||
m := lo + (hi-lo)/2
|
||||
if hi-lo > 40 {
|
||||
s := (hi - lo) / 8
|
||||
medianOfThree_func(data, lo, lo+s, lo+2*s)
|
||||
medianOfThree_func(data, m, m-s, m+s)
|
||||
medianOfThree_func(data, hi-1, hi-1-s, hi-1-2*s)
|
||||
}
|
||||
medianOfThree_func(data, lo, m, hi-1)
|
||||
pivot := lo
|
||||
a, c := lo+1, hi-1
|
||||
for ; a < c && data.Less(a, pivot); a++ {
|
||||
}
|
||||
b := a
|
||||
for {
|
||||
for ; b < c && !data.Less(pivot, b); b++ {
|
||||
}
|
||||
for ; b < c && data.Less(pivot, c-1); c-- {
|
||||
}
|
||||
if b >= c {
|
||||
break
|
||||
}
|
||||
data.Swap(b, c-1)
|
||||
b++
|
||||
c--
|
||||
}
|
||||
protect := hi-c < 5
|
||||
if !protect && hi-c < (hi-lo)/4 {
|
||||
dups := 0
|
||||
if !data.Less(pivot, hi-1) {
|
||||
data.Swap(c, hi-1)
|
||||
c++
|
||||
dups++
|
||||
}
|
||||
if !data.Less(b-1, pivot) {
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
if !data.Less(m, pivot) {
|
||||
data.Swap(m, b-1)
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
protect = dups > 1
|
||||
}
|
||||
if protect {
|
||||
for {
|
||||
for ; a < b && !data.Less(b-1, pivot); b-- {
|
||||
}
|
||||
for ; a < b && data.Less(a, pivot); a++ {
|
||||
}
|
||||
if a >= b {
|
||||
break
|
||||
}
|
||||
data.Swap(a, b-1)
|
||||
a++
|
||||
b--
|
||||
}
|
||||
}
|
||||
data.Swap(pivot, b-1)
|
||||
return b - 1, c
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:quickSort
|
||||
func quickSort_func(data lessSwap, a, b, maxDepth int) {
|
||||
for b-a > 12 {
|
||||
if maxDepth == 0 {
|
||||
heapSort_func(data, a, b)
|
||||
return
|
||||
}
|
||||
maxDepth--
|
||||
mlo, mhi := doPivot_func(data, a, b)
|
||||
if mlo-a < b-mhi {
|
||||
quickSort_func(data, a, mlo, maxDepth)
|
||||
a = mhi
|
||||
} else {
|
||||
quickSort_func(data, mhi, b, maxDepth)
|
||||
b = mlo
|
||||
}
|
||||
}
|
||||
if b-a > 1 {
|
||||
for i := a + 6; i < b; i++ {
|
||||
if data.Less(i, i-6) {
|
||||
data.Swap(i, i-6)
|
||||
}
|
||||
}
|
||||
insertionSort_func(data, a, b)
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:stable
|
||||
func stable_func(data lessSwap, n int) {
|
||||
blockSize := 20
|
||||
a, b := 0, blockSize
|
||||
for b <= n {
|
||||
insertionSort_func(data, a, b)
|
||||
a = b
|
||||
b += blockSize
|
||||
}
|
||||
insertionSort_func(data, a, n)
|
||||
for blockSize < n {
|
||||
a, b = 0, 2*blockSize
|
||||
for b <= n {
|
||||
symMerge_func(data, a, a+blockSize, b)
|
||||
a = b
|
||||
b += 2 * blockSize
|
||||
}
|
||||
if m := a + blockSize; m < n {
|
||||
symMerge_func(data, a, m, n)
|
||||
}
|
||||
blockSize *= 2
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:symMerge
|
||||
func symMerge_func(data lessSwap, a, m, b int) {
|
||||
if m-a == 1 {
|
||||
i := m
|
||||
j := b
|
||||
for i < j {
|
||||
h := i + (j-i)/2
|
||||
if data.Less(h, a) {
|
||||
i = h + 1
|
||||
} else {
|
||||
j = h
|
||||
}
|
||||
}
|
||||
for k := a; k < i-1; k++ {
|
||||
data.Swap(k, k+1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if b-m == 1 {
|
||||
i := a
|
||||
j := m
|
||||
for i < j {
|
||||
h := i + (j-i)/2
|
||||
if !data.Less(m, h) {
|
||||
i = h + 1
|
||||
} else {
|
||||
j = h
|
||||
}
|
||||
}
|
||||
for k := m; k > i; k-- {
|
||||
data.Swap(k, k-1)
|
||||
}
|
||||
return
|
||||
}
|
||||
mid := a + (b-a)/2
|
||||
n := mid + m
|
||||
var start, r int
|
||||
if m > mid {
|
||||
start = n - b
|
||||
r = mid
|
||||
} else {
|
||||
start = a
|
||||
r = m
|
||||
}
|
||||
p := n - 1
|
||||
for start < r {
|
||||
c := start + (r-start)/2
|
||||
if !data.Less(p-c, c) {
|
||||
start = c + 1
|
||||
} else {
|
||||
r = c
|
||||
}
|
||||
}
|
||||
end := n - start
|
||||
if start < m && m < end {
|
||||
rotate_func(data, start, m, end)
|
||||
}
|
||||
if a < start && start < mid {
|
||||
symMerge_func(data, a, start, mid)
|
||||
}
|
||||
if mid < end && end < b {
|
||||
symMerge_func(data, mid, end, b)
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-generated variant of sort.go:rotate
|
||||
func rotate_func(data lessSwap, a, m, b int) {
|
||||
i := m - a
|
||||
j := b - m
|
||||
for i != j {
|
||||
if i > j {
|
||||
swapRange_func(data, m-i, m, j)
|
||||
i -= j
|
||||
} else {
|
||||
swapRange_func(data, m-i, m+j-i, i)
|
||||
j -= i
|
||||
}
|
||||
}
|
||||
swapRange_func(data, m-i, m, i)
|
||||
}
|
||||
@ -1,39 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package strutil
|
||||
|
||||
var internStr = map[string]string{}
|
||||
|
||||
// RegisterCommonString adds common strings to the interned string
|
||||
// table. This should be called during init from the main
|
||||
// goroutine, not later at runtime.
|
||||
func RegisterCommonString(s ...string) {
|
||||
for _, v := range s {
|
||||
internStr[v] = v
|
||||
}
|
||||
}
|
||||
|
||||
// StringFromBytes returns string(v), minimizing copies for common values of v
|
||||
// as previously registered with RegisterCommonString.
|
||||
func StringFromBytes(v []byte) string {
|
||||
// In Go 1.3, this string conversion in the map lookup does not allocate
|
||||
// to make a new string. We depend on Go 1.3, so this is always free:
|
||||
if s, ok := internStr[string(v)]; ok {
|
||||
return s
|
||||
}
|
||||
return string(v)
|
||||
}
|
||||
@ -1,117 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package strutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// ParseUintBytes is like strconv.ParseUint, but using a []byte.
|
||||
func ParseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) {
|
||||
var cutoff, maxVal uint64
|
||||
|
||||
if bitSize == 0 {
|
||||
bitSize = int(strconv.IntSize)
|
||||
}
|
||||
|
||||
s0 := s
|
||||
switch {
|
||||
case len(s) < 1:
|
||||
err = strconv.ErrSyntax
|
||||
goto Error
|
||||
|
||||
case 2 <= base && base <= 36:
|
||||
// valid base; nothing to do
|
||||
|
||||
case base == 0:
|
||||
// Look for octal, hex prefix.
|
||||
switch {
|
||||
case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'):
|
||||
base = 16
|
||||
s = s[2:]
|
||||
if len(s) < 1 {
|
||||
err = strconv.ErrSyntax
|
||||
goto Error
|
||||
}
|
||||
case s[0] == '0':
|
||||
base = 8
|
||||
default:
|
||||
base = 10
|
||||
}
|
||||
|
||||
default:
|
||||
err = errors.New("invalid base " + strconv.Itoa(base))
|
||||
goto Error
|
||||
}
|
||||
|
||||
n = 0
|
||||
cutoff = cutoff64(base)
|
||||
maxVal = 1<<uint(bitSize) - 1
|
||||
|
||||
for i := 0; i < len(s); i++ {
|
||||
var v byte
|
||||
d := s[i]
|
||||
switch {
|
||||
case '0' <= d && d <= '9':
|
||||
v = d - '0'
|
||||
case 'a' <= d && d <= 'z':
|
||||
v = d - 'a' + 10
|
||||
case 'A' <= d && d <= 'Z':
|
||||
v = d - 'A' + 10
|
||||
default:
|
||||
n = 0
|
||||
err = strconv.ErrSyntax
|
||||
goto Error
|
||||
}
|
||||
if int(v) >= base {
|
||||
n = 0
|
||||
err = strconv.ErrSyntax
|
||||
goto Error
|
||||
}
|
||||
|
||||
if n >= cutoff {
|
||||
// n*base overflows
|
||||
n = 1<<64 - 1
|
||||
err = strconv.ErrRange
|
||||
goto Error
|
||||
}
|
||||
n *= uint64(base)
|
||||
|
||||
n1 := n + uint64(v)
|
||||
if n1 < n || n1 > maxVal {
|
||||
// n+v overflows
|
||||
n = 1<<64 - 1
|
||||
err = strconv.ErrRange
|
||||
goto Error
|
||||
}
|
||||
n = n1
|
||||
}
|
||||
|
||||
return n, nil
|
||||
|
||||
Error:
|
||||
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
|
||||
}
|
||||
|
||||
// Return the first number n such that n*base >= 1<<64.
|
||||
func cutoff64(base int) uint64 {
|
||||
if base < 2 {
|
||||
return 0
|
||||
}
|
||||
return (1<<64-1)/uint64(base) + 1
|
||||
}
|
||||
@ -1,200 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package strutil contains string and byte processing functions.
|
||||
package strutil // import "go4.org/strutil"
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Fork of Go's implementation in pkg/strings/strings.go:
|
||||
// Generic split: splits after each instance of sep,
|
||||
// including sepSave bytes of sep in the subarrays.
|
||||
func genSplit(dst []string, s, sep string, sepSave, n int) []string {
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
if sep == "" {
|
||||
panic("sep is empty")
|
||||
}
|
||||
if n < 0 {
|
||||
n = strings.Count(s, sep) + 1
|
||||
}
|
||||
c := sep[0]
|
||||
start := 0
|
||||
na := 0
|
||||
for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ {
|
||||
if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
|
||||
dst = append(dst, s[start:i+sepSave])
|
||||
na++
|
||||
start = i + len(sep)
|
||||
i += len(sep) - 1
|
||||
}
|
||||
}
|
||||
dst = append(dst, s[start:])
|
||||
return dst
|
||||
}
|
||||
|
||||
// AppendSplitN is like strings.SplitN but appends to and returns dst.
|
||||
// Unlike strings.SplitN, an empty separator is not supported.
|
||||
// The count n determines the number of substrings to return:
|
||||
// n > 0: at most n substrings; the last substring will be the unsplit remainder.
|
||||
// n == 0: the result is nil (zero substrings)
|
||||
// n < 0: all substrings
|
||||
func AppendSplitN(dst []string, s, sep string, n int) []string {
|
||||
return genSplit(dst, s, sep, 0, n)
|
||||
}
|
||||
|
||||
// equalFoldRune compares a and b runes whether they fold equally.
|
||||
//
|
||||
// The code comes from strings.EqualFold, but shortened to only one rune.
|
||||
func equalFoldRune(sr, tr rune) bool {
|
||||
if sr == tr {
|
||||
return true
|
||||
}
|
||||
// Make sr < tr to simplify what follows.
|
||||
if tr < sr {
|
||||
sr, tr = tr, sr
|
||||
}
|
||||
// Fast check for ASCII.
|
||||
if tr < utf8.RuneSelf && 'A' <= sr && sr <= 'Z' {
|
||||
// ASCII, and sr is upper case. tr must be lower case.
|
||||
if tr == sr+'a'-'A' {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// General case. SimpleFold(x) returns the next equivalent rune > x
|
||||
// or wraps around to smaller values.
|
||||
r := unicode.SimpleFold(sr)
|
||||
for r != sr && r < tr {
|
||||
r = unicode.SimpleFold(r)
|
||||
}
|
||||
if r == tr {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// HasPrefixFold is like strings.HasPrefix but uses Unicode case-folding.
|
||||
func HasPrefixFold(s, prefix string) bool {
|
||||
if prefix == "" {
|
||||
return true
|
||||
}
|
||||
for _, pr := range prefix {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
// step with s, too
|
||||
sr, size := utf8.DecodeRuneInString(s)
|
||||
if sr == utf8.RuneError {
|
||||
return false
|
||||
}
|
||||
s = s[size:]
|
||||
if !equalFoldRune(sr, pr) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// HasSuffixFold is like strings.HasPrefix but uses Unicode case-folding.
|
||||
func HasSuffixFold(s, suffix string) bool {
|
||||
if suffix == "" {
|
||||
return true
|
||||
}
|
||||
// count the runes and bytes in s, but only till rune count of suffix
|
||||
bo, so := len(s), len(suffix)
|
||||
for bo > 0 && so > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(s[:bo])
|
||||
if r == utf8.RuneError {
|
||||
return false
|
||||
}
|
||||
bo -= size
|
||||
|
||||
sr, size := utf8.DecodeLastRuneInString(suffix[:so])
|
||||
if sr == utf8.RuneError {
|
||||
return false
|
||||
}
|
||||
so -= size
|
||||
|
||||
if !equalFoldRune(r, sr) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return so == 0
|
||||
}
|
||||
|
||||
// ContainsFold is like strings.Contains but uses Unicode case-folding.
|
||||
func ContainsFold(s, substr string) bool {
|
||||
if substr == "" {
|
||||
return true
|
||||
}
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
firstRune := rune(substr[0])
|
||||
if firstRune >= utf8.RuneSelf {
|
||||
firstRune, _ = utf8.DecodeRuneInString(substr)
|
||||
}
|
||||
for i, rune := range s {
|
||||
if equalFoldRune(rune, firstRune) && HasPrefixFold(s[i:], substr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsPlausibleJSON reports whether s likely contains a JSON object, without
|
||||
// actually parsing it. It's meant to be a light heuristic.
|
||||
func IsPlausibleJSON(s string) bool {
|
||||
return startsWithOpenBrace(s) && endsWithCloseBrace(s)
|
||||
}
|
||||
|
||||
func isASCIIWhite(b byte) bool { return b == ' ' || b == '\n' || b == '\r' || b == '\t' }
|
||||
|
||||
func startsWithOpenBrace(s string) bool {
|
||||
for len(s) > 0 {
|
||||
switch {
|
||||
case s[0] == '{':
|
||||
return true
|
||||
case isASCIIWhite(s[0]):
|
||||
s = s[1:]
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func endsWithCloseBrace(s string) bool {
|
||||
for len(s) > 0 {
|
||||
last := len(s) - 1
|
||||
switch {
|
||||
case s[last] == '}':
|
||||
return true
|
||||
case isASCIIWhite(s[last]):
|
||||
s = s[:last]
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package syncutil
|
||||
|
||||
// A Gate limits concurrency.
|
||||
type Gate struct {
|
||||
c chan struct{}
|
||||
}
|
||||
|
||||
// NewGate returns a new gate that will only permit max operations at once.
|
||||
func NewGate(max int) *Gate {
|
||||
return &Gate{make(chan struct{}, max)}
|
||||
}
|
||||
|
||||
// Start starts an operation, blocking until the gate has room.
|
||||
func (g *Gate) Start() {
|
||||
g.c <- struct{}{}
|
||||
}
|
||||
|
||||
// Done finishes an operation.
|
||||
func (g *Gate) Done() {
|
||||
select {
|
||||
case <-g.c:
|
||||
default:
|
||||
panic("Done called more than Start")
|
||||
}
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package syncutil
|
||||
|
||||
import "sync"
|
||||
|
||||
// A Group is like a sync.WaitGroup and coordinates doing
|
||||
// multiple things at once. Its zero value is ready to use.
|
||||
type Group struct {
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex // guards errs
|
||||
errs []error
|
||||
}
|
||||
|
||||
// Go runs fn in its own goroutine, but does not wait for it to complete.
|
||||
// Call Err or Errs to wait for all the goroutines to complete.
|
||||
func (g *Group) Go(fn func() error) {
|
||||
g.wg.Add(1)
|
||||
go func() {
|
||||
defer g.wg.Done()
|
||||
err := fn()
|
||||
if err != nil {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.errs = append(g.errs, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait waits for all the previous calls to Go to complete.
|
||||
func (g *Group) Wait() {
|
||||
g.wg.Wait()
|
||||
}
|
||||
|
||||
// Err waits for all previous calls to Go to complete and returns the
|
||||
// first non-nil error, or nil.
|
||||
func (g *Group) Err() error {
|
||||
g.wg.Wait()
|
||||
if len(g.errs) > 0 {
|
||||
return g.errs[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Errs waits for all previous calls to Go to complete and returns
|
||||
// all non-nil errors.
|
||||
func (g *Group) Errs() []error {
|
||||
g.wg.Wait()
|
||||
return g.errs
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
/*
|
||||
Copyright 2014 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package syncutil
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// A Once will perform a successful action exactly once.
|
||||
//
|
||||
// Unlike a sync.Once, this Once's func returns an error
|
||||
// and is re-armed on failure.
|
||||
type Once struct {
|
||||
m sync.Mutex
|
||||
done uint32
|
||||
}
|
||||
|
||||
// Do calls the function f if and only if Do has not been invoked
|
||||
// without error for this instance of Once. In other words, given
|
||||
// var once Once
|
||||
// if once.Do(f) is called multiple times, only the first call will
|
||||
// invoke f, even if f has a different value in each invocation unless
|
||||
// f returns an error. A new instance of Once is required for each
|
||||
// function to execute.
|
||||
//
|
||||
// Do is intended for initialization that must be run exactly once. Since f
|
||||
// is niladic, it may be necessary to use a function literal to capture the
|
||||
// arguments to a function to be invoked by Do:
|
||||
// err := config.once.Do(func() error { return config.init(filename) })
|
||||
func (o *Once) Do(f func() error) error {
|
||||
if atomic.LoadUint32(&o.done) == 1 {
|
||||
return nil
|
||||
}
|
||||
// Slow-path.
|
||||
o.m.Lock()
|
||||
defer o.m.Unlock()
|
||||
var err error
|
||||
if o.done == 0 {
|
||||
err = f()
|
||||
if err == nil {
|
||||
atomic.StoreUint32(&o.done, 1)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
package syncutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOnce(t *testing.T) {
|
||||
timesRan := 0
|
||||
f := func() error {
|
||||
timesRan++
|
||||
return nil
|
||||
}
|
||||
|
||||
once := Once{}
|
||||
grp := Group{}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
grp.Go(func() error { return once.Do(f) })
|
||||
}
|
||||
|
||||
if grp.Err() != nil {
|
||||
t.Errorf("Expected no errors, got %v", grp.Err())
|
||||
}
|
||||
|
||||
if timesRan != 1 {
|
||||
t.Errorf("Expected to run one time, ran %d", timesRan)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOnceErroring verifies we retry on every error, but stop after
|
||||
// the first success.
|
||||
func TestOnceErroring(t *testing.T) {
|
||||
timesRan := 0
|
||||
f := func() error {
|
||||
timesRan++
|
||||
if timesRan < 3 {
|
||||
return errors.New("retry")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
once := Once{}
|
||||
grp := Group{}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
grp.Go(func() error { return once.Do(f) })
|
||||
}
|
||||
|
||||
if len(grp.Errs()) != 2 {
|
||||
t.Errorf("Expected two errors, got %d", len(grp.Errs()))
|
||||
}
|
||||
|
||||
if timesRan != 3 {
|
||||
t.Errorf("Expected to run two times, ran %d", timesRan)
|
||||
}
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
package syncutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type debugT bool
|
||||
|
||||
var debug = debugT(false)
|
||||
|
||||
func (d debugT) Printf(format string, args ...interface{}) {
|
||||
if bool(d) {
|
||||
log.Printf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Sem implements a semaphore that can have multiple units acquired/released
|
||||
// at a time.
|
||||
type Sem struct {
|
||||
c *sync.Cond // Protects size
|
||||
max, free int64
|
||||
}
|
||||
|
||||
// NewSem creates a semaphore with max units available for acquisition.
|
||||
func NewSem(max int64) *Sem {
|
||||
return &Sem{
|
||||
c: sync.NewCond(new(sync.Mutex)),
|
||||
free: max,
|
||||
max: max,
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire will deduct n units from the semaphore. If the deduction would
|
||||
// result in the available units falling below zero, the call will block until
|
||||
// another go routine returns units via a call to Release. If more units are
|
||||
// requested than the semaphore is configured to hold, error will be non-nil.
|
||||
func (s *Sem) Acquire(n int64) error {
|
||||
if n > s.max {
|
||||
return fmt.Errorf("sem: attempt to acquire more units than semaphore size %d > %d", n, s.max)
|
||||
}
|
||||
s.c.L.Lock()
|
||||
defer s.c.L.Unlock()
|
||||
for {
|
||||
debug.Printf("Acquire check max %d free %d, n %d", s.max, s.free, n)
|
||||
if s.free >= n {
|
||||
s.free -= n
|
||||
return nil
|
||||
}
|
||||
debug.Printf("Acquire Wait max %d free %d, n %d", s.max, s.free, n)
|
||||
s.c.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// Release will return n units to the semaphore and notify any currently
|
||||
// blocking Acquire calls.
|
||||
func (s *Sem) Release(n int64) {
|
||||
s.c.L.Lock()
|
||||
defer s.c.L.Unlock()
|
||||
debug.Printf("Release max %d free %d, n %d", s.max, s.free, n)
|
||||
s.free += n
|
||||
s.c.Broadcast()
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
package syncutil_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"go4.org/syncutil"
|
||||
)
|
||||
|
||||
func TestSem(t *testing.T) {
|
||||
s := syncutil.NewSem(5)
|
||||
|
||||
if err := s.Acquire(2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Acquire(2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
s.Release(2)
|
||||
s.Release(2)
|
||||
}()
|
||||
if err := s.Acquire(5); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemErr(t *testing.T) {
|
||||
s := syncutil.NewSem(5)
|
||||
if err := s.Acquire(6); err == nil {
|
||||
t.Fatal("Didn't get expected error for large acquire.")
|
||||
}
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package singleflight provides a duplicate function call suppression
|
||||
// mechanism.
|
||||
package singleflight // import "go4.org/syncutil/singleflight"
|
||||
|
||||
import "sync"
|
||||
|
||||
// call is an in-flight or completed Do call
|
||||
type call struct {
|
||||
wg sync.WaitGroup
|
||||
val interface{}
|
||||
err error
|
||||
}
|
||||
|
||||
// Group represents a class of work and forms a namespace in which
|
||||
// units of work can be executed with duplicate suppression.
|
||||
type Group struct {
|
||||
mu sync.Mutex // protects m
|
||||
m map[string]*call // lazily initialized
|
||||
}
|
||||
|
||||
// Do executes and returns the results of the given function, making
|
||||
// sure that only one execution is in-flight for a given key at a
|
||||
// time. If a duplicate comes in, the duplicate caller waits for the
|
||||
// original to complete and receives the same results.
|
||||
func (g *Group) Do(key string, fn func() (interface{}, error)) (interface{}, error) {
|
||||
g.mu.Lock()
|
||||
if g.m == nil {
|
||||
g.m = make(map[string]*call)
|
||||
}
|
||||
if c, ok := g.m[key]; ok {
|
||||
g.mu.Unlock()
|
||||
c.wg.Wait()
|
||||
return c.val, c.err
|
||||
}
|
||||
c := new(call)
|
||||
c.wg.Add(1)
|
||||
g.m[key] = c
|
||||
g.mu.Unlock()
|
||||
|
||||
c.val, c.err = fn()
|
||||
c.wg.Done()
|
||||
|
||||
g.mu.Lock()
|
||||
delete(g.m, key)
|
||||
g.mu.Unlock()
|
||||
|
||||
return c.val, c.err
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
/*
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package singleflight
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDo(t *testing.T) {
|
||||
var g Group
|
||||
v, err := g.Do("key", func() (interface{}, error) {
|
||||
return "bar", nil
|
||||
})
|
||||
if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want {
|
||||
t.Errorf("Do = %v; want %v", got, want)
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("Do error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoErr(t *testing.T) {
|
||||
var g Group
|
||||
someErr := errors.New("Some error")
|
||||
v, err := g.Do("key", func() (interface{}, error) {
|
||||
return nil, someErr
|
||||
})
|
||||
if err != someErr {
|
||||
t.Errorf("Do error = %v; want someErr %v", err, someErr)
|
||||
}
|
||||
if v != nil {
|
||||
t.Errorf("unexpected non-nil value %#v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoDupSuppress(t *testing.T) {
|
||||
var g Group
|
||||
c := make(chan string)
|
||||
var calls int32
|
||||
fn := func() (interface{}, error) {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return <-c, nil
|
||||
}
|
||||
|
||||
const n = 10
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < n; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
v, err := g.Do("key", fn)
|
||||
if err != nil {
|
||||
t.Errorf("Do error: %v", err)
|
||||
}
|
||||
if v.(string) != "bar" {
|
||||
t.Errorf("got %q; want %q", v, "bar")
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond) // let goroutines above block
|
||||
c <- "bar"
|
||||
wg.Wait()
|
||||
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||
t.Errorf("number of calls = %d; want 1", got)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue