alt-orchestra/talos
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support type filter in list API and CLI

Browse Source 
Closes #2068.

Signed-off-by: Alexey Palazhchenko <alexey.palazhchenko@gmail.com>
This commit is contained in:
Alexey Palazhchenko 2020-12-23 23:05:12 +03:00 committed by talos-bot
parent 5590fe19eb
commit f3465b8e3e
11 changed files with 1405 additions and 1185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
api/machine/machine.proto
View File

@ -276,22 +276,34 @@ message CopyRequest {
string root_path = 1;
}
// ListRequest describes a request to list the contents of a directory
// ListRequest describes a request to list the contents of a directory.
message ListRequest {
// Root indicates the root directory for the list. If not indicated, '/' is
// Root indicates the root directory for the list. If not indicated, '/' is
// presumed.
string root = 1;
// Recurse indicates that subdirectories should be recursed.
bool recurse = 2;
// RecursionDepth indicates how many levels of subdirectories should be
// recursed. The default (0) indicates that no limit should be enforced.
// recursed. The default (0) indicates that no limit should be enforced.
int32 recursion_depth = 3;
// File type.
enum Type {
// Regular file (not directory, symlink, etc).
REGULAR = 0;
// Directory.
DIRECTORY = 1;
// Symbolic link.
SYMLINK = 2;
}
// Types indicates what file type should be returned. If not indicated,
// all files will be returned.
repeated Type types = 4;
}
// DiskUsageRequest describes a request to list disk usage of directories and regular files
message DiskUsageRequest {
// RecursionDepth indicates how many levels of subdirectories should be
// recursed. The default (0) indicates that no limit should be enforced.
// recursed. The default (0) indicates that no limit should be enforced.
int32 recursion_depth = 1;
// All write sizes for all files, not just directories.
bool all = 2;

29
cmd/talosctl/cmd/talos/list.go
View File

@ -9,6 +9,7 @@ import (
"fmt"
"io"
"os"
"strings"
"text/tabwriter"
"time"
@ -28,6 +29,7 @@ var (
recurse bool
recursionDepth int32
humanizeFlag bool
types []string
)
// lsCmd represents the ls command.
@ -45,10 +47,29 @@ var lsCmd = &cobra.Command{
rootDir = args[0]
}
// handle all variants: --type=f,l; -tfl; etc
var reqTypes []machineapi.ListRequest_Type
for _, typ := range types {
for _, t := range typ {
// handle both `find -type X` and os.FileMode.String() designations
switch t {
case 'f':
reqTypes = append(reqTypes, machineapi.ListRequest_REGULAR)
case 'd':
reqTypes = append(reqTypes, machineapi.ListRequest_DIRECTORY)
case 'l', 'L':
reqTypes = append(reqTypes, machineapi.ListRequest_SYMLINK)
default:
return fmt.Errorf("invalid file type: %s", string(t))
}
}
}
stream, err := c.LS(ctx, &machineapi.ListRequest{
Root: rootDir,
Recurse: recurse,
RecursionDepth: recursionDepth,
Types: reqTypes,
})
if err != nil {
return fmt.Errorf("error fetching logs: %s", err)
@ -172,9 +193,17 @@ var lsCmd = &cobra.Command{
}
func init() {
typesHelp := strings.Join([]string{
"filter by specified types:",
"f" + "\t" + "regular file",
"d" + "\t" + "directory",
"l, L" + "\t" + "symbolic link",
}, "\n")
lsCmd.Flags().BoolVarP(&long, "long", "l", false, "display additional file details")
lsCmd.Flags().BoolVarP(&recurse, "recurse", "r", false, "recurse into subdirectories")
lsCmd.Flags().BoolVarP(&humanizeFlag, "humanize", "H", false, "humanize size and time in the output")
lsCmd.Flags().Int32VarP(&recursionDepth, "depth", "d", 0, "maximum recursion depth")
lsCmd.Flags().StringSliceVarP(&types, "type", "t", nil, typesHelp)
addCommand(lsCmd)
}

27
internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go
View File

@ -667,6 +667,8 @@ func (s *Server) Copy(req *machine.CopyRequest, obj machine.MachineService_CopyS
}
// List implements the machine.MachineServer interface.
//
//nolint:gocyclo
func (s *Server) List(req *machine.ListRequest, obj machine.MachineService_ListServer) error {
if req == nil {
req = new(machine.ListRequest)
@ -682,17 +684,34 @@ func (s *Server) List(req *machine.ListRequest, obj machine.MachineService_ListS
req.Root = "/"
}
var maxDepth int
var opts []archiver.WalkerOption
if req.Recurse {
if req.RecursionDepth == 0 {
maxDepth = -1
opts = append(opts, archiver.WithMaxRecurseDepth(-1))
} else {
maxDepth = int(req.RecursionDepth)
opts = append(opts, archiver.WithMaxRecurseDepth(int(req.RecursionDepth)))
}
}
files, err := archiver.Walker(obj.Context(), req.Root, archiver.WithMaxRecurseDepth(maxDepth))
if len(req.Types) > 0 {
types := make([]archiver.FileType, 0, len(req.Types))
for _, t := range req.Types {
switch t {
case machine.ListRequest_REGULAR:
types = append(types, archiver.RegularFileType)
case machine.ListRequest_DIRECTORY:
types = append(types, archiver.DirectoryFileType)
case machine.ListRequest_SYMLINK:
types = append(types, archiver.SymlinkFileType)
}
}
opts = append(opts, archiver.WithFileTypes(types...))
}
files, err := archiver.Walker(obj.Context(), req.Root, opts...)
if err != nil {
return err
}

2
pkg/archiver/archiver.go
View File

@ -2,7 +2,7 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Package archiver provides a service to archive part of the filesystem into tar archive
// Package archiver provides a service to archive part of the filesystem into tar archive.
package archiver
import (

3
pkg/archiver/archiver_test.go
View File

@ -2,7 +2,6 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Package archiver provides a service to archive part of the filesystem into tar archive
package archiver_test
import (
@ -38,7 +37,7 @@ var filesFixture = []struct {
},
{
Path: "/dev/random",
Mode: 0o600 | os.ModeCharDevice,
Mode: 0o600 | os.ModeDevice | os.ModeCharDevice,
},
{
Path: "/usr/bin/cp",

1
pkg/archiver/tar_test.go
View File

@ -2,7 +2,6 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Package archiver provides a service to archive part of the filesystem into tar archive
package archiver_test
import (

51
pkg/archiver/walker.go
View File

@ -20,10 +20,21 @@ type FileItem struct {
Error error
}
// FileType is a file type.
type FileType int
// File types.
const (
RegularFileType FileType = iota
DirectoryFileType
SymlinkFileType
)
type walkerOptions struct {
skipRoot bool
maxRecurseDepth int
fnmatchPatterns []string
types map[FileType]struct{}
}
// WalkerOption configures Walker.
@ -54,7 +65,19 @@ func WithFnmatchPatterns(patterns ...string) WalkerOption {
}
}
// Walker provides a channel of file info/paths for archival
// WithFileTypes filters results by file types.
//
// Default is not to do any filtering.
func WithFileTypes(fileType ...FileType) WalkerOption {
return func(o *walkerOptions) {
o.types = make(map[FileType]struct{}, len(fileType))
for _, t := range fileType {
o.types[t] = struct{}{}
}
}
}
// Walker provides a channel of file info/paths for archival.
//
//nolint: gocyclo
func Walker(ctx context.Context, rootPath string, options ...WalkerOption) (<-chan FileItem, error) {
@ -96,6 +119,30 @@ func Walker(ctx context.Context, rootPath string, options ...WalkerOption) (<-ch
item.RelPath, item.Error = filepath.Rel(rootPath, path)
}
// TODO: refactor all those `if item.Error == nil &&` conditions
if item.Error == nil && len(opts.types) > 0 {
var matches bool
for t := range opts.types {
switch t {
case RegularFileType:
matches = fileInfo.Mode()&os.ModeType == 0
case DirectoryFileType:
matches = fileInfo.Mode()&os.ModeDir != 0
case SymlinkFileType:
matches = fileInfo.Mode()&os.ModeSymlink != 0
}
if matches {
break
}
}
if !matches {
return nil
}
}
if item.Error == nil && path == rootPath && opts.skipRoot && fileInfo.IsDir() {
// skip containing directory
return nil
@ -106,7 +153,7 @@ func Walker(ctx context.Context, rootPath string, options ...WalkerOption) (<-ch
}
if item.Error == nil && len(opts.fnmatchPatterns) > 0 {
matches := false
var matches bool
for _, pattern := range opts.fnmatchPatterns {
if matches, _ = filepath.Match(pattern, item.RelPath); matches { //nolint: errcheck

29
pkg/archiver/walker_test.go
View File

@ -2,7 +2,6 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Package archiver provides a service to archive part of the filesystem into tar archive
package archiver_test
import (
@ -104,12 +103,21 @@ func (suite *WalkerSuite) TestIterationSymlink() {
err := os.Mkdir(original, 0o755)
suite.Require().NoError(err)
newname := filepath.Join(suite.tmpDir, "new")
defer func() {
err = os.RemoveAll(original)
suite.Require().NoError(err)
}()
// NB: We make this a relative symlink to make the test more complete.
newname := filepath.Join(suite.tmpDir, "new")
err = os.Symlink("original", newname)
suite.Require().NoError(err)
defer func() {
err = os.Remove(newname)
suite.Require().NoError(err)
}()
err = ioutil.WriteFile(filepath.Join(original, "original.txt"), []byte{}, 0o666)
suite.Require().NoError(err)
@ -131,6 +139,23 @@ func (suite *WalkerSuite) TestIterationNotFound() {
suite.Require().Error(err)
}
func (suite *WalkerSuite) TestIterationTypes() {
ch, err := archiver.Walker(context.Background(), suite.tmpDir, archiver.WithFileTypes(archiver.DirectoryFileType))
suite.Require().NoError(err)
relPaths := []string(nil)
for fi := range ch {
suite.Require().NoError(fi.Error)
relPaths = append(relPaths, fi.RelPath)
}
suite.Assert().Equal([]string{
".", "dev", "etc", "etc/certs", "lib", "usr", "usr/bin",
},
relPaths)
}
func TestWalkerSuite(t *testing.T) {
suite.Run(t, new(WalkerSuite))
}

2397
pkg/machinery/api/machine/machine.pb.go
View File

File diff suppressed because it is too large Load Diff

17
website/content/docs/v0.8/Reference/api.md
View File

@ -149,6 +149,7 @@ description: Talos gRPC API reference.
- [VersionInfo](#machine.VersionInfo)
- [VersionResponse](#machine.VersionResponse)
- [ListRequest.Type](#machine.ListRequest.Type)
- [MachineConfig.MachineType](#machine.MachineConfig.MachineType)
- [PhaseEvent.Action](#machine.PhaseEvent.Action)
- [RecoverRequest.Source](#machine.RecoverRequest.Source)
@ -1205,7 +1206,7 @@ GenerateConfiguration describes the response to a generate configuration request
<a name="machine.ListRequest"></a>
### ListRequest
ListRequest describes a request to list the contents of a directory
ListRequest describes a request to list the contents of a directory.
| Field | Type | Label | Description |
@ -1213,6 +1214,7 @@ ListRequest describes a request to list the contents of a directory
| root | [string](#string) | | Root indicates the root directory for the list. If not indicated, '/' is presumed. |
| recurse | [bool](#bool) | | Recurse indicates that subdirectories should be recursed. |
| recursion_depth | [int32](#int32) | | RecursionDepth indicates how many levels of subdirectories should be recursed. The default (0) indicates that no limit should be enforced. |
| types | [ListRequest.Type](#machine.ListRequest.Type) | repeated | Types indicates what file type should be returned. If not indicated, all files will be returned. |
@ -2500,6 +2502,19 @@ rpc upgrade
<!-- end messages -->
<a name="machine.ListRequest.Type"></a>
### ListRequest.Type
File type.
| Name | Number | Description |
| ---- | ------ | ----------- |
| REGULAR | 0 | Regular file (not directory, symlink, etc). |
| DIRECTORY | 1 | Directory. |
| SYMLINK | 2 | Symbolic link. |
<a name="machine.MachineConfig.MachineType"></a>
### MachineConfig.MachineType

14
website/content/docs/v0.8/Reference/cli.md
View File

@ -1149,11 +1149,15 @@ talosctl list [path] [flags]
### Options
```
-d, --depth int32 maximum recursion depth
-h, --help help for list
-H, --humanize humanize size and time in the output
-l, --long display additional file details
-r, --recurse recurse into subdirectories
-d, --depth int32 maximum recursion depth
-h, --help help for list
-H, --humanize humanize size and time in the output
-l, --long display additional file details
-r, --recurse recurse into subdirectories
-t, --type strings filter by specified types:
f regular file
d directory
l, L symbolic link
```
### Options inherited from parent commands