package llb import ( "context" _ "crypto/sha256" // for opencontainers/go-digest "os" "path" "strconv" "strings" "time" "github.com/moby/buildkit/solver/pb" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) // Examples: // local := llb.Local(...) // llb.Image().Dir("/abc").File(Mkdir("./foo").Mkfile("/abc/foo/bar", []byte("data"))) // llb.Image().File(Mkdir("/foo").Mkfile("/foo/bar", []byte("data"))) // llb.Image().File(Copy(local, "/foo", "/bar")).File(Copy(local, "/foo2", "/bar2")) // // a := Mkdir("./foo") // *FileAction /ced/foo // b := Mkdir("./bar") // /abc/bar // c := b.Copy(a.WithState(llb.Scratch().Dir("/ced")), "./foo", "./baz") // /abc/baz // llb.Image().Dir("/abc").File(c) // // In future this can be extended to multiple outputs with: // a := Mkdir("./foo") // b, id := a.GetSelector() // c := b.Mkdir("./bar") // filestate = state.File(c) // filestate.GetOutput(id).Exec() func NewFileOp(s State, action *FileAction, c Constraints) *FileOp { action = action.bind(s) f := &FileOp{ action: action, constraints: c, } f.output = &output{vertex: f, getIndex: func() (pb.OutputIndex, error) { return pb.OutputIndex(0), nil }} return f } // CopyInput is either llb.State or *FileActionWithState type CopyInput interface { isFileOpCopyInput() } type subAction interface { toProtoAction(context.Context, string, pb.InputIndex) (pb.IsFileAction, error) } type capAdder interface { addCaps(*FileOp) } type FileAction struct { state *State prev *FileAction action subAction err error } func (fa *FileAction) Mkdir(p string, m os.FileMode, opt ...MkdirOption) *FileAction { a := Mkdir(p, m, opt...) a.prev = fa return a } func (fa *FileAction) Mkfile(p string, m os.FileMode, dt []byte, opt ...MkfileOption) *FileAction { a := Mkfile(p, m, dt, opt...) a.prev = fa return a } func (fa *FileAction) Rm(p string, opt ...RmOption) *FileAction { a := Rm(p, opt...) a.prev = fa return a } func (fa *FileAction) Copy(input CopyInput, src, dest string, opt ...CopyOption) *FileAction { a := Copy(input, src, dest, opt...) a.prev = fa return a } func (fa *FileAction) allOutputs(m map[Output]struct{}) { if fa == nil { return } if fa.state != nil && fa.state.Output() != nil { m[fa.state.Output()] = struct{}{} } if a, ok := fa.action.(*fileActionCopy); ok { if a.state != nil { if out := a.state.Output(); out != nil { m[out] = struct{}{} } } else if a.fas != nil { a.fas.allOutputs(m) } } fa.prev.allOutputs(m) } func (fa *FileAction) bind(s State) *FileAction { if fa == nil { return nil } fa2 := *fa fa2.prev = fa.prev.bind(s) fa2.state = &s return &fa2 } func (fa *FileAction) WithState(s State) CopyInput { return &fileActionWithState{FileAction: fa.bind(s)} } type fileActionWithState struct { *FileAction } func (fas *fileActionWithState) isFileOpCopyInput() {} func Mkdir(p string, m os.FileMode, opt ...MkdirOption) *FileAction { var mi MkdirInfo for _, o := range opt { o.SetMkdirOption(&mi) } return &FileAction{ action: &fileActionMkdir{ file: p, mode: m, info: mi, }, } } type fileActionMkdir struct { file string mode os.FileMode info MkdirInfo } func (a *fileActionMkdir) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) { return &pb.FileAction_Mkdir{ Mkdir: &pb.FileActionMkDir{ Path: normalizePath(parent, a.file, false), Mode: int32(a.mode & 0777), MakeParents: a.info.MakeParents, Owner: a.info.ChownOpt.marshal(base), Timestamp: marshalTime(a.info.CreatedTime), }, }, nil } type MkdirOption interface { SetMkdirOption(*MkdirInfo) } type ChownOption interface { MkdirOption MkfileOption CopyOption } type mkdirOptionFunc func(*MkdirInfo) func (fn mkdirOptionFunc) SetMkdirOption(mi *MkdirInfo) { fn(mi) } var _ MkdirOption = &MkdirInfo{} func WithParents(b bool) MkdirOption { return mkdirOptionFunc(func(mi *MkdirInfo) { mi.MakeParents = b }) } type MkdirInfo struct { MakeParents bool ChownOpt *ChownOpt CreatedTime *time.Time } func (mi *MkdirInfo) SetMkdirOption(mi2 *MkdirInfo) { *mi2 = *mi } func WithUser(name string) ChownOption { opt := ChownOpt{} parts := strings.SplitN(name, ":", 2) for i, v := range parts { switch i { case 0: uid, err := parseUID(v) if err != nil { opt.User = &UserOpt{Name: v} } else { opt.User = &UserOpt{UID: uid} } case 1: gid, err := parseUID(v) if err != nil { opt.Group = &UserOpt{Name: v} } else { opt.Group = &UserOpt{UID: gid} } } } return opt } func parseUID(str string) (int, error) { if str == "root" { return 0, nil } uid, err := strconv.ParseInt(str, 10, 32) if err != nil { return 0, err } return int(uid), nil } func WithUIDGID(uid, gid int) ChownOption { return ChownOpt{ User: &UserOpt{UID: uid}, Group: &UserOpt{UID: gid}, } } type ChownOpt struct { User *UserOpt Group *UserOpt } func (co ChownOpt) SetMkdirOption(mi *MkdirInfo) { mi.ChownOpt = &co } func (co ChownOpt) SetMkfileOption(mi *MkfileInfo) { mi.ChownOpt = &co } func (co ChownOpt) SetCopyOption(mi *CopyInfo) { mi.ChownOpt = &co } func (co *ChownOpt) marshal(base pb.InputIndex) *pb.ChownOpt { if co == nil { return nil } return &pb.ChownOpt{ User: co.User.marshal(base), Group: co.Group.marshal(base), } } type UserOpt struct { UID int Name string } func (up *UserOpt) marshal(base pb.InputIndex) *pb.UserOpt { if up == nil { return nil } if up.Name != "" { return &pb.UserOpt{User: &pb.UserOpt_ByName{ByName: &pb.NamedUserOpt{ Name: up.Name, Input: base}}} } return &pb.UserOpt{User: &pb.UserOpt_ByID{ByID: uint32(up.UID)}} } func Mkfile(p string, m os.FileMode, dt []byte, opts ...MkfileOption) *FileAction { var mi MkfileInfo for _, o := range opts { o.SetMkfileOption(&mi) } return &FileAction{ action: &fileActionMkfile{ file: p, mode: m, dt: dt, info: mi, }, } } type MkfileOption interface { SetMkfileOption(*MkfileInfo) } type MkfileInfo struct { ChownOpt *ChownOpt CreatedTime *time.Time } func (mi *MkfileInfo) SetMkfileOption(mi2 *MkfileInfo) { *mi2 = *mi } var _ MkfileOption = &MkfileInfo{} type fileActionMkfile struct { file string mode os.FileMode dt []byte info MkfileInfo } func (a *fileActionMkfile) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) { return &pb.FileAction_Mkfile{ Mkfile: &pb.FileActionMkFile{ Path: normalizePath(parent, a.file, false), Mode: int32(a.mode & 0777), Data: a.dt, Owner: a.info.ChownOpt.marshal(base), Timestamp: marshalTime(a.info.CreatedTime), }, }, nil } func Rm(p string, opts ...RmOption) *FileAction { var mi RmInfo for _, o := range opts { o.SetRmOption(&mi) } return &FileAction{ action: &fileActionRm{ file: p, info: mi, }, } } type RmOption interface { SetRmOption(*RmInfo) } type rmOptionFunc func(*RmInfo) func (fn rmOptionFunc) SetRmOption(mi *RmInfo) { fn(mi) } type RmInfo struct { AllowNotFound bool AllowWildcard bool } func (mi *RmInfo) SetRmOption(mi2 *RmInfo) { *mi2 = *mi } var _ RmOption = &RmInfo{} func WithAllowNotFound(b bool) RmOption { return rmOptionFunc(func(mi *RmInfo) { mi.AllowNotFound = b }) } func WithAllowWildcard(b bool) RmOption { return rmOptionFunc(func(mi *RmInfo) { mi.AllowWildcard = b }) } type fileActionRm struct { file string info RmInfo } func (a *fileActionRm) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) { return &pb.FileAction_Rm{ Rm: &pb.FileActionRm{ Path: normalizePath(parent, a.file, false), AllowNotFound: a.info.AllowNotFound, AllowWildcard: a.info.AllowWildcard, }, }, nil } func Copy(input CopyInput, src, dest string, opts ...CopyOption) *FileAction { var state *State var fas *fileActionWithState var err error if st, ok := input.(State); ok { state = &st } else if v, ok := input.(*fileActionWithState); ok { fas = v } else { err = errors.Errorf("invalid input type %T for copy", input) } var mi CopyInfo for _, o := range opts { o.SetCopyOption(&mi) } return &FileAction{ action: &fileActionCopy{ state: state, fas: fas, src: src, dest: dest, info: mi, }, err: err, } } type CopyOption interface { SetCopyOption(*CopyInfo) } type CopyInfo struct { Mode *os.FileMode FollowSymlinks bool CopyDirContentsOnly bool IncludePatterns []string ExcludePatterns []string AttemptUnpack bool CreateDestPath bool AllowWildcard bool AllowEmptyWildcard bool ChownOpt *ChownOpt CreatedTime *time.Time } func (mi *CopyInfo) SetCopyOption(mi2 *CopyInfo) { *mi2 = *mi } var _ CopyOption = &CopyInfo{} type fileActionCopy struct { state *State fas *fileActionWithState src string dest string info CopyInfo } func (a *fileActionCopy) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) { src, err := a.sourcePath(ctx) if err != nil { return nil, err } c := &pb.FileActionCopy{ Src: src, Dest: normalizePath(parent, a.dest, true), Owner: a.info.ChownOpt.marshal(base), IncludePatterns: a.info.IncludePatterns, ExcludePatterns: a.info.ExcludePatterns, AllowWildcard: a.info.AllowWildcard, AllowEmptyWildcard: a.info.AllowEmptyWildcard, FollowSymlink: a.info.FollowSymlinks, DirCopyContents: a.info.CopyDirContentsOnly, AttemptUnpackDockerCompatibility: a.info.AttemptUnpack, CreateDestPath: a.info.CreateDestPath, Timestamp: marshalTime(a.info.CreatedTime), } if a.info.Mode != nil { c.Mode = int32(*a.info.Mode) } else { c.Mode = -1 } return &pb.FileAction_Copy{ Copy: c, }, nil } func (a *fileActionCopy) sourcePath(ctx context.Context) (string, error) { p := path.Clean(a.src) if !path.IsAbs(p) { if a.state != nil { dir, err := a.state.GetDir(ctx) if err != nil { return "", err } p = path.Join("/", dir, p) } else if a.fas != nil { dir, err := a.fas.state.GetDir(ctx) if err != nil { return "", err } p = path.Join("/", dir, p) } } return p, nil } func (a *fileActionCopy) addCaps(f *FileOp) { if len(a.info.IncludePatterns) != 0 || len(a.info.ExcludePatterns) != 0 { addCap(&f.constraints, pb.CapFileCopyIncludeExcludePatterns) } } type CreatedTime time.Time func WithCreatedTime(t time.Time) CreatedTime { return CreatedTime(t) } func (c CreatedTime) SetMkdirOption(mi *MkdirInfo) { mi.CreatedTime = (*time.Time)(&c) } func (c CreatedTime) SetMkfileOption(mi *MkfileInfo) { mi.CreatedTime = (*time.Time)(&c) } func (c CreatedTime) SetCopyOption(mi *CopyInfo) { mi.CreatedTime = (*time.Time)(&c) } func marshalTime(t *time.Time) int64 { if t == nil { return -1 } return t.UnixNano() } type FileOp struct { MarshalCache action *FileAction output Output constraints Constraints isValidated bool } func (f *FileOp) Validate(context.Context) error { if f.isValidated { return nil } if f.action == nil { return errors.Errorf("action is required") } f.isValidated = true return nil } type marshalState struct { ctx context.Context visited map[*FileAction]*fileActionState inputs []*pb.Input actions []*fileActionState } func newMarshalState(ctx context.Context) *marshalState { return &marshalState{ visited: map[*FileAction]*fileActionState{}, ctx: ctx, } } type fileActionState struct { base pb.InputIndex input pb.InputIndex inputRelative *int input2 pb.InputIndex input2Relative *int target int action subAction fa *FileAction } func (ms *marshalState) addInput(st *fileActionState, c *Constraints, o Output) (pb.InputIndex, error) { inp, err := o.ToInput(ms.ctx, c) if err != nil { return 0, err } for i, inp2 := range ms.inputs { if *inp == *inp2 { return pb.InputIndex(i), nil } } i := pb.InputIndex(len(ms.inputs)) ms.inputs = append(ms.inputs, inp) return i, nil } func (ms *marshalState) add(fa *FileAction, c *Constraints) (*fileActionState, error) { if st, ok := ms.visited[fa]; ok { return st, nil } if fa.err != nil { return nil, fa.err } var prevState *fileActionState if parent := fa.prev; parent != nil { var err error prevState, err = ms.add(parent, c) if err != nil { return nil, err } } st := &fileActionState{ action: fa.action, input: -1, input2: -1, base: -1, fa: fa, } if source := fa.state.Output(); source != nil { inp, err := ms.addInput(st, c, source) if err != nil { return nil, err } st.base = inp } if fa.prev == nil { st.input = st.base } else { st.inputRelative = &prevState.target } if a, ok := fa.action.(*fileActionCopy); ok { if a.state != nil { if out := a.state.Output(); out != nil { inp, err := ms.addInput(st, c, out) if err != nil { return nil, err } st.input2 = inp } } else if a.fas != nil { src, err := ms.add(a.fas.FileAction, c) if err != nil { return nil, err } st.input2Relative = &src.target } else { return nil, errors.Errorf("invalid empty source for copy") } } st.target = len(ms.actions) ms.visited[fa] = st ms.actions = append(ms.actions, st) return st, nil } func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) { if f.Cached(c) { return f.Load() } if err := f.Validate(ctx); err != nil { return "", nil, nil, nil, err } addCap(&f.constraints, pb.CapFileBase) pfo := &pb.FileOp{} if f.constraints.Platform == nil { p, err := getPlatform(*f.action.state)(ctx) if err != nil { return "", nil, nil, nil, err } f.constraints.Platform = p } pop, md := MarshalConstraints(c, &f.constraints) pop.Op = &pb.Op_File{ File: pfo, } state := newMarshalState(ctx) _, err := state.add(f.action, c) if err != nil { return "", nil, nil, nil, err } pop.Inputs = state.inputs for i, st := range state.actions { if adder, isCapAdder := st.action.(capAdder); isCapAdder { adder.addCaps(f) } output := pb.OutputIndex(-1) if i+1 == len(state.actions) { output = 0 } var parent string if st.fa.state != nil { parent, err = st.fa.state.GetDir(ctx) if err != nil { return "", nil, nil, nil, err } } action, err := st.action.toProtoAction(ctx, parent, st.base) if err != nil { return "", nil, nil, nil, err } pfo.Actions = append(pfo.Actions, &pb.FileAction{ Input: getIndex(st.input, len(state.inputs), st.inputRelative), SecondaryInput: getIndex(st.input2, len(state.inputs), st.input2Relative), Output: output, Action: action, }) } dt, err := pop.Marshal() if err != nil { return "", nil, nil, nil, err } f.Store(dt, md, f.constraints.SourceLocations, c) return f.Load() } func normalizePath(parent, p string, keepSlash bool) string { origPath := p p = path.Clean(p) if !path.IsAbs(p) { p = path.Join("/", parent, p) } if keepSlash { if strings.HasSuffix(origPath, "/") && !strings.HasSuffix(p, "/") { p += "/" } else if strings.HasSuffix(origPath, "/.") { if p != "/" { p += "/" } p += "." } } return p } func (f *FileOp) Output() Output { return f.output } func (f *FileOp) Inputs() (inputs []Output) { mm := map[Output]struct{}{} f.action.allOutputs(mm) for o := range mm { inputs = append(inputs, o) } return inputs } func getIndex(input pb.InputIndex, len int, relative *int) pb.InputIndex { if relative != nil { return pb.InputIndex(len + *relative) } return input }