Skip to content

Commit 0445045

Browse files
committed
Initial merge of shelltools into datatools
1 parent eee87c5 commit 0445045

33 files changed

+2732
-18
lines changed

Makefile

+63-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ VERSION = $(shell grep -m1 'Version = ' $(PROJECT).go | cut -d\" -f 2)
77

88
BRANCH = $(shell git branch | grep '* ' | cut -d\ -f 2)
99

10-
build: bin/csvcols bin/csvrows bin/csvfind bin/csvjoin bin/jsoncols bin/jsonrange bin/xlsx2json bin/xlsx2csv bin/csv2mdtable bin/csv2xlsx bin/csv2json bin/vcard2json bin/jsonmunge
10+
build: bin/csvcols bin/csvrows bin/csvfind bin/csvjoin bin/jsoncols bin/jsonrange bin/xlsx2json bin/xlsx2csv bin/csv2mdtable bin/csv2xlsx bin/csv2json bin/vcard2json bin/jsonmunge bin/findfile bin/finddir bin/mergepath bin/reldate bin/range bin/timefmt bin/urlparse
11+
1112

1213
bin/csvcols: datatools.go cmds/csvcols/csvcols.go
1314
go build -o bin/csvcols cmds/csvcols/csvcols.go
@@ -48,6 +49,27 @@ bin/vcard2json: datatools.go cmds/vcard2json/vcard2json.go
4849
bin/jsonmunge: datatools.go cmds/jsonmunge/jsonmunge.go
4950
go build -o bin/jsonmunge cmds/jsonmunge/jsonmunge.go
5051

52+
bin/findfile: datatools.go cmds/findfile/findfile.go
53+
go build -o bin/findfile cmds/findfile/findfile.go
54+
55+
bin/finddir: datatools.go cmds/finddir/finddir.go
56+
go build -o bin/finddir cmds/finddir/finddir.go
57+
58+
bin/mergepath: datatools.go cmds/mergepath/mergepath.go
59+
go build -o bin/mergepath cmds/mergepath/mergepath.go
60+
61+
bin/reldate: datatools.go cmds/reldate/reldate.go
62+
go build -o bin/reldate cmds/reldate/reldate.go
63+
64+
bin/range: datatools.go cmds/range/range.go
65+
go build -o bin/range cmds/range/range.go
66+
67+
bin/timefmt: datatools.go cmds/timefmt/timefmt.go
68+
go build -o bin/timefmt cmds/timefmt/timefmt.go
69+
70+
bin/urlparse: datatools.go cmds/urlparse/urlparse.go
71+
go build -o bin/urlparse cmds/urlparse/urlparse.go
72+
5173

5274
test:
5375
go test
@@ -88,6 +110,13 @@ install:
88110
env GOBIN=$(HOME)/bin go install cmds/csv2json/csv2json.go
89111
env GOBIN=$(HOME)/bin go install cmds/vcard2json/vcard2json.go
90112
env GOBIN=$(HOME)/bin go install cmds/jsonmunge/jsonmunge.go
113+
env GOBIN=$(HOME)/bin go install cmds/findfile/findfile.go
114+
env GOBIN=$(HOME)/bin go install cmds/finddir/finddir.go
115+
env GOBIN=$(HOME)/bin go install cmds/mergepath/mergepath.go
116+
env GOBIN=$(HOME)/bin go install cmds/reldate/reldate.go
117+
env GOBIN=$(HOME)/bin go install cmds/range/range.go
118+
env GOBIN=$(HOME)/bin go install cmds/timefmt/timefmt.go
119+
env GOBIN=$(HOME)/bin go install cmds/urlparse/urlparse.go
91120

92121
dist/linux-amd64:
93122
mkdir -p dist/bin
@@ -104,9 +133,17 @@ dist/linux-amd64:
104133
env GOOS=linux GOARCH=amd64 go build -o dist/bin/csv2json cmds/csv2json/csv2json.go
105134
env GOOS=linux GOARCH=amd64 go build -o dist/bin/jsonmunge cmds/jsonmunge/jsonmunge.go
106135
env GOOS=linux GOARCH=amd64 go build -o dist/bin/vcard2json cmds/vcard2json/vcard2json.go
136+
env GOOS=linux GOARCH=amd64 go build -o dist/bin/findfile cmds/findfile/findfile.go
137+
env GOOS=linux GOARCH=amd64 go build -o dist/bin/finddir cmds/finddir/finddir.go
138+
env GOOS=linux GOARCH=amd64 go build -o dist/bin/mergepath cmds/mergepath/mergepath.go
139+
env GOOS=linux GOARCH=amd64 go build -o dist/bin/reldate cmds/reldate/reldate.go
140+
env GOOS=linux GOARCH=amd64 go build -o dist/bin/range cmds/range/range.go
141+
env GOOS=linux GOARCH=amd64 go build -o dist/bin/timefmt cmds/timefmt/timefmt.go
142+
env GOOS=linux GOARCH=amd64 go build -o dist/bin/urlparse cmds/urlparse/urlparse.go
107143
cd dist && zip -r $(PROJECT)-$(VERSION)-linux-amd64.zip README.md LICENSE INSTALL.md bin/* docs/* how-to/* demos/*
108144
rm -fR dist/bin
109145

146+
110147
dist/macosx-amd64:
111148
mkdir -p dist/bin
112149
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/csvcols cmds/csvcols/csvcols.go
@@ -122,8 +159,17 @@ dist/macosx-amd64:
122159
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/csv2json cmds/csv2json/csv2json.go
123160
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/vcard2json cmds/vcard2json/vcard2json.go
124161
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/jsonmunge cmds/jsonmunge/jsonmunge.go
162+
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/findfile cmds/findfile/findfile.go
163+
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/finddir cmds/finddir/finddir.go
164+
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/mergepath cmds/mergepath/mergepath.go
165+
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/reldate cmds/reldate/reldate.go
166+
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/range cmds/range/range.go
167+
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/timefmt cmds/timefmt/timefmt.go
168+
env GOOS=darwin GOARCH=amd64 go build -o dist/bin/urlparse cmds/urlparse/urlparse.go
125169
cd dist && zip -r $(PROJECT)-$(VERSION)-macosx-amd64.zip README.md LICENSE INSTALL.md bin/* docs/* how-to/* demos/*
126170
rm -fR dist/bin
171+
172+
127173

128174
dist/windows-amd64:
129175
mkdir -p dist/bin
@@ -140,10 +186,19 @@ dist/windows-amd64:
140186
env GOOS=windows GOARCH=amd64 go build -o dist/bin/csv2json.exe cmds/csv2json/csv2json.go
141187
env GOOS=windows GOARCH=amd64 go build -o dist/bin/vcard2json.exe cmds/vcard2json/vcard2json.go
142188
env GOOS=windows GOARCH=amd64 go build -o dist/bin/jsonmunge.exe cmds/jsonmunge/jsonmunge.go
189+
env GOOS=windows GOARCH=amd64 go build -o dist/bin/findfile.exe cmds/findfile/findfile.go
190+
env GOOS=windows GOARCH=amd64 go build -o dist/bin/finddir.exe cmds/finddir/finddir.go
191+
env GOOS=windows GOARCH=amd64 go build -o dist/bin/mergepath.exe cmds/mergepath/mergepath.go
192+
env GOOS=windows GOARCH=amd64 go build -o dist/bin/reldate.exe cmds/reldate/reldate.go
193+
env GOOS=windows GOARCH=amd64 go build -o dist/bin/range.exe cmds/range/range.go
194+
env GOOS=windows GOARCH=amd64 go build -o dist/bin/timefmt.exe cmds/timefmt/timefmt.go
195+
env GOOS=windows GOARCH=amd64 go build -o dist/bin/urlparse.exe cmds/urlparse/urlparse.go
143196
cd dist && zip -r $(PROJECT)-$(VERSION)-windows-amd64.zip README.md LICENSE INSTALL.md bin/* docs/* how-to/* demos/*
144197
rm -fR dist/bin
145198

146199

200+
201+
147202
dist/raspbian-arm7:
148203
mkdir -p dist/bin
149204
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/csvcols cmds/csvcols/csvcols.go
@@ -159,6 +214,13 @@ dist/raspbian-arm7:
159214
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/csv2json cmds/csv2json/csv2json.go
160215
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/vcard2json cmds/vcard2json/vcard2json.go
161216
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/jsonmunge cmds/jsonmunge/jsonmunge.go
217+
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/findfile cmds/findfile/findfile.go
218+
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/finddir cmds/finddir/finddir.go
219+
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/mergepath cmds/mergepath/mergepath.go
220+
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/reldate cmds/reldate/reldate.go
221+
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/range cmds/range/range.go
222+
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/timefmt cmds/timefmt/timefmt.go
223+
env GOOS=linux GOARCH=arm GOARM=7 go build -o dist/bin/urlparse cmds/urlparse/urlparse.go
162224
cd dist && zip -r $(PROJECT)-$(VERSION)-raspbian-arm7.zip README.md LICENSE INSTALL.md bin/* docs/* how-to/* demos/*
163225
rm -fR dist/bin
164226

README.md

+33-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11

22
# datatools
33

4-
Command line utilities for simplifying work with CSV, JSON, Excel Workbooks and plain text files or content.
4+
Command line utilities for simplifying work with CSV, JSON, Excel Workbooks and plain text files or content. It also now includes
5+
shelltools which are utilities useful for shell scripting.
56

67
+ [csvcols](docs/csvcols.html) - a tool for formatting command line arguments into CSV row of columns or filtering CSV rows for specific columns
78
+ [csvrows](docs/csvrows.html) - a tool for formatting command line arguments into CSV columns of rows or filtering CSV columns for specific rows
@@ -36,3 +37,34 @@ Or see [INSTALL.md](install.html) for details for installing
3637
compiled versions of the programs.
3738

3839

40+
41+
## shelltools
42+
43+
Various utilities for simplifying work on the command line.
44+
45+
+ [findfile](docs/findfile.html) - find files based on prefix, suffix or contained string
46+
+ [finddir](docs/finddir.html) - find directories based on prefix, suffix or contained string
47+
+ [mergepath](docs/mergepath.html) - prefix, append, clip path variables
48+
+ [range](docs/range.html) - emit a range of integers (useful for numbered loops in Bash)
49+
+ [reldate](docs/reldate.html) - display a relative date in YYYY-MM-DD format
50+
+ [timefmt](docs/timefmt.html) - format a time value based on Golang's time format language
51+
+ [urlparse](docs/urlparse.html) - split a URL into parts
52+
53+
Compiled versions are provided for Linux (amd64), Mac OS X (amd64),
54+
Windows 10 (amd64) and Raspbian (ARM7). See https://github.com/caltechlibrary/shelltools/releases.
55+
56+
Use the utilities try "-help" option for a full list of options.
57+
58+
59+
## Installation
60+
61+
_shelltools_ is go get-able.
62+
63+
```
64+
go get github.com/caltechlibrary/shelltools/...
65+
```
66+
67+
Or see [INSTALL.md](install.html) for details for installing
68+
compiled versions of the programs.
69+
70+

cmds/finddir/finddir.go

+202
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
//
2+
// finddir.go - a simple directory tree walker that looks for directories
3+
// by name, basename or extension. Basically a unix "find" light to
4+
// demonstrate walking the file system
5+
//
6+
// @author R. S. Doiel, <[email protected]>
7+
//
8+
// Copyright (c) 2017, Caltech
9+
// All rights not granted herein are expressly reserved by Caltech.
10+
//
11+
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
12+
//
13+
// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
14+
//
15+
// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
16+
//
17+
// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
18+
//
19+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
20+
//
21+
package main
22+
23+
import (
24+
"flag"
25+
"fmt"
26+
"log"
27+
"os"
28+
"path"
29+
"path/filepath"
30+
"strings"
31+
"time"
32+
33+
// CaltechLibrary packages
34+
"github.com/caltechlibrary/cli"
35+
"github.com/caltechlibrary/shelltools"
36+
)
37+
38+
var (
39+
usage = `USAGE: %s [OPTIONS] [TARGET] [DIRECTORIES_TO_SEARCH]`
40+
41+
description = `
42+
SYNOPSIS
43+
44+
%s finds directory based on matching prefix, suffix or contained text in base filename.
45+
`
46+
47+
examples = `
48+
EXAMPLE
49+
50+
%s -p img
51+
52+
Find all subdirectories starting with "img".
53+
`
54+
55+
// Standard Options
56+
showHelp bool
57+
showVersion bool
58+
showLicense bool
59+
60+
// Application Options
61+
showModificationTime bool
62+
findPrefix bool
63+
findContains bool
64+
findSuffix bool
65+
findAll bool
66+
stopOnErrors bool
67+
outputFullPath bool
68+
optDepth int
69+
pathSep string
70+
)
71+
72+
func display(docroot, p string, m time.Time) {
73+
var s string
74+
if outputFullPath == true {
75+
s, _ = filepath.Abs(p)
76+
} else {
77+
s, _ = filepath.Rel(docroot, p)
78+
}
79+
if showModificationTime == true {
80+
fmt.Printf("%s %s\n", m.Format("2006-01-02 15:04:05 -0700"), s)
81+
return
82+
}
83+
fmt.Printf("%s\n", s)
84+
}
85+
86+
func walkPath(docroot string, target string) error {
87+
return filepath.Walk(docroot, func(p string, info os.FileInfo, err error) error {
88+
if err != nil {
89+
if stopOnErrors == true {
90+
return fmt.Errorf("Can't read %s, %s", p, err)
91+
}
92+
return nil
93+
}
94+
// If a directory then apply rules for display
95+
if info.Mode().IsDir() == true {
96+
if optDepth > 0 {
97+
currentDepth := strings.Count(p, pathSep) + 1
98+
if currentDepth > optDepth {
99+
return filepath.SkipDir
100+
}
101+
}
102+
//s := filepath.Base(p)
103+
s := p
104+
105+
switch {
106+
case findPrefix == true && strings.HasPrefix(s, target) == true:
107+
display(docroot, p, info.ModTime())
108+
case findSuffix == true && strings.HasSuffix(s, target) == true:
109+
display(docroot, p, info.ModTime())
110+
case findContains == true && strings.Contains(s, target) == true:
111+
display(docroot, p, info.ModTime())
112+
case strings.Compare(s, target) == 0:
113+
display(docroot, p, info.ModTime())
114+
case findAll == true:
115+
display(docroot, p, info.ModTime())
116+
}
117+
}
118+
return nil
119+
})
120+
}
121+
122+
func init() {
123+
// Standard Options
124+
flag.BoolVar(&showHelp, "h", false, "display this help message")
125+
flag.BoolVar(&showHelp, "help", false, "display this help message")
126+
flag.BoolVar(&showLicense, "l", false, "display license information")
127+
flag.BoolVar(&showLicense, "license", false, "display license information")
128+
flag.BoolVar(&showVersion, "v", false, "display version message")
129+
flag.BoolVar(&showVersion, "version", false, "display version message")
130+
131+
// Application Specific Options
132+
flag.BoolVar(&showModificationTime, "m", false, "display file modification time before the path")
133+
flag.BoolVar(&showModificationTime, "mod-time", false, "display file modification time before the path")
134+
flag.BoolVar(&stopOnErrors, "e", false, "Stop walk on file system errors (e.g. permissions)")
135+
flag.BoolVar(&stopOnErrors, "error-stop", false, "Stop walk on file system errors (e.g. permissions)")
136+
flag.BoolVar(&findPrefix, "p", false, "find file(s) based on basename prefix")
137+
flag.BoolVar(&findPrefix, "prefix", false, "find file(s) based on basename prefix")
138+
flag.BoolVar(&findContains, "c", false, "find file(s) based on basename containing text")
139+
flag.BoolVar(&findContains, "contains", false, "find file(s) based on basename containing text")
140+
flag.BoolVar(&findSuffix, "s", false, "find file(s) based on basename suffix")
141+
flag.BoolVar(&findSuffix, "suffix", false, "find file(s) based on basename suffix")
142+
flag.BoolVar(&outputFullPath, "f", false, "list full path for files found")
143+
flag.BoolVar(&outputFullPath, "full-path", false, "list full path for files found")
144+
flag.IntVar(&optDepth, "d", 0, "Limit depth of directories walked")
145+
flag.IntVar(&optDepth, "depth", 0, "Limit depth of directories walked")
146+
pathSep = string(os.PathSeparator)
147+
}
148+
149+
func main() {
150+
appName := path.Base(os.Args[0])
151+
flag.Parse()
152+
args := flag.Args()
153+
154+
// Configuration and command line interation
155+
cfg := cli.New(appName, appName, fmt.Sprintf(shelltools.LicenseText, appName, shelltools.Version), shelltools.Version)
156+
cfg.UsageText = fmt.Sprintf(usage, appName)
157+
cfg.DescriptionText = fmt.Sprintf(description, appName)
158+
cfg.ExampleText = fmt.Sprintf(examples, appName)
159+
160+
if showHelp == true {
161+
fmt.Println(cfg.Usage())
162+
os.Exit(0)
163+
}
164+
165+
if showLicense == true {
166+
fmt.Println(cfg.License())
167+
os.Exit(0)
168+
}
169+
170+
if showVersion == true {
171+
fmt.Println(cfg.Version())
172+
os.Exit(0)
173+
}
174+
175+
if findPrefix == false && findSuffix == false && findContains == false {
176+
findAll = true
177+
}
178+
179+
if findAll == false && len(args) == 0 {
180+
fmt.Println(cfg.Usage())
181+
os.Exit(1)
182+
}
183+
184+
// Shift the target directory name so args holds the directories to search
185+
target := ""
186+
if findAll == false && len(args) > 0 {
187+
target, args = args[0], args[1:]
188+
}
189+
190+
// Handle the case where currect work directory is assumed
191+
if len(args) == 0 {
192+
args = []string{"."}
193+
}
194+
195+
// For each directory to search walk the path
196+
for _, dir := range args {
197+
err := walkPath(dir, target)
198+
if err != nil {
199+
log.Fatal(err)
200+
}
201+
}
202+
}

0 commit comments

Comments
 (0)