Skip to content

Commit 539b7ca

Browse files
Ericson2314Ericson2314
committed
WIP rework goals
1 parent 6ca3cbf commit 539b7ca

13 files changed

+491
-96
lines changed

‎src/libstore/build/build-trace-goal.cc‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
#include "nix/store/build/build-trace-goal.hh"
2+
#include "nix/util/finally.hh"
3+
#include "nix/store/build/worker.hh"
4+
#include "nix/store/build/substitution-goal.hh"
5+
#include "nix/util/callback.hh"
6+
#include "nix/util/util.hh"
7+
#include "nix/store/derivations.hh"
8+
#include "nix/store/store-open.hh"
9+
#include "nix/store/build/derivation-resolution-goal.hh"
10+
11+
namespace nix {
12+
13+
BuildTraceGoal::BuildTraceGoal(const SingleDerivedPath::Built & id, Worker & worker)
14+
: Goal{worker, init()}
15+
, id{id}
16+
{
17+
name = fmt("substitution of '%s'", id.to_string(worker.store));
18+
trace("created");
19+
}
20+
21+
Goal::Co BuildTraceGoal::init()
22+
{
23+
trace("init");
24+
25+
DrvOutput id2{
26+
.drvPath = StorePath::dummy,
27+
.outputName = id.output,
28+
};
29+
30+
// No `std::visit` with coroutines :(
31+
if (const auto * path = std::get_if<SingleDerivedPath::Opaque>(&*id.drvPath)) {
32+
// At least we know the drv path statically, can procede
33+
id2.drvPath = path->path;
34+
} else if (const auto * outputDeriving = std::get_if<SingleDerivedPath::Built>(&*id.drvPath)) {
35+
// Dynamic derivation case, need to resolve that first.
36+
37+
auto g = worker.makeBuildTraceGoal({
38+
outputDeriving->drvPath,
39+
outputDeriving->output,
40+
});
41+
42+
co_await await(Goals{upcast_goal(g)});
43+
44+
if (nrFailed > 0) {
45+
debug("The output deriving path '%s' could not be resolved", outputDeriving->to_string(worker.store));
46+
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
47+
}
48+
49+
id2.drvPath = g->outputInfo->outPath;
50+
}
51+
52+
/* If the derivation already exists, we’re done */
53+
if ((outputInfo = worker.store.queryRealisation(id2))) {
54+
co_return amDone(ecSuccess);
55+
}
56+
57+
/**
58+
* Firstly, whether we know the status, secondly, what it is
59+
*/
60+
std::optional<bool> drvIsResolved;
61+
62+
/* If the derivation has statically-known output paths */
63+
if (worker.evalStore.isValidPath(id2.drvPath)) {
64+
auto drv = worker.evalStore.readDerivation(id2.drvPath);
65+
auto os = drv.outputsAndOptPaths(worker.store);
66+
/* Mark what we now know */
67+
drvIsResolved = {drv.inputDrvs.map.empty()};
68+
if (auto * p = get(os, id2.outputName)) {
69+
if (auto & outPath = p->second) {
70+
outputInfo = std::make_shared<UnkeyedRealisation>(*outPath);
71+
co_return amDone(ecSuccess);
72+
} else {
73+
/* Otherwise, not failure, just looking up build trace below. */
74+
}
75+
} else {
76+
debug(
77+
"Derivation '%s' does not have output '%s', impossible to find build trace key-value pair",
78+
worker.store.printStorePath(id2.drvPath),
79+
id2.outputName);
80+
co_return amDone(ecFailed);
81+
}
82+
}
83+
84+
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
85+
86+
bool substituterFailed = false;
87+
88+
if (!drvIsResolved || *drvIsResolved) {
89+
/* Since derivation might be resolved --- isn't known to be
90+
not-resolved, it might have entries. So, let's try querying
91+
the substituters. */
92+
for (const auto & sub : subs) {
93+
trace("trying next substituter");
94+
95+
/* The callback of the curl download below can outlive `this` (if
96+
some other error occurs), so it must not touch `this`. So put
97+
the shared state in a separate refcounted object. */
98+
auto outPipe = std::make_shared<MuxablePipe>();
99+
#ifndef _WIN32
100+
outPipe->create();
101+
#else
102+
outPipe->createAsyncPipe(worker.ioport.get());
103+
#endif
104+
105+
auto promise = std::make_shared<std::promise<std::shared_ptr<const UnkeyedRealisation>>>();
106+
107+
sub->queryRealisation(
108+
id2, {[outPipe(outPipe), promise(promise)](std::future<std::shared_ptr<const UnkeyedRealisation>> res) {
109+
try {
110+
Finally updateStats([&]() { outPipe->writeSide.close(); });
111+
promise->set_value(res.get());
112+
} catch (...) {
113+
promise->set_exception(std::current_exception());
114+
}
115+
}});
116+
117+
worker.childStarted(
118+
shared_from_this(),
119+
{
120+
#ifndef _WIN32
121+
outPipe->readSide.get()
122+
#else
123+
&*outPipe
124+
#endif
125+
},
126+
true,
127+
false);
128+
129+
co_await Suspend{};
130+
131+
worker.childTerminated(this);
132+
133+
std::shared_ptr<const UnkeyedRealisation> outputInfo;
134+
try {
135+
outputInfo = promise->get_future().get();
136+
} catch (std::exception & e) {
137+
printError(e.what());
138+
substituterFailed = true;
139+
}
140+
141+
if (!outputInfo)
142+
continue;
143+
144+
worker.store.registerDrvOutput({*outputInfo, id2});
145+
146+
trace("finished");
147+
co_return amDone(ecSuccess);
148+
}
149+
}
150+
151+
/* Derivation might not be resolved, let's try doing that */
152+
trace("trying resolving derivation in build-trace goal");
153+
154+
auto g = worker.makeDerivationResolutionGoal(id2.drvPath);
155+
156+
co_await await(Goals{g});
157+
158+
if (nrFailed > 0) {
159+
/* None left. Terminate this goal and let someone else deal
160+
with it. */
161+
debug(
162+
"derivation output '%s' is required, but there is no substituter that can provide it",
163+
id2.render(worker.store));
164+
165+
if (substituterFailed) {
166+
worker.failedSubstitutions++;
167+
worker.updateProgress();
168+
}
169+
170+
/* Hack: don't indicate failure if there were no substituters.
171+
In that case the calling derivation should just do a
172+
build. */
173+
co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters);
174+
}
175+
176+
/* This should be set if the goal succeeded */
177+
assert(g->resolvedDrv);
178+
179+
/* Try everything again, now with a resolved derivation */
180+
auto bt2 = worker.makeBuildTraceGoal({
181+
makeConstantStorePathRef(g->resolvedDrvPath),
182+
id2.outputName,
183+
});
184+
185+
co_await await(Goals{bt2});
186+
187+
/* Set the build trace value as our own. Note the signure will not
188+
match our key since we're the unresolved derivation, but that's
189+
fine. We're not writing it to the DB; that's `bt2`' job. */
190+
if (bt2->outputInfo)
191+
outputInfo = bt2->outputInfo;
192+
193+
co_return amDone(bt2->exitCode, bt2->ex);
194+
}
195+
196+
std::string BuildTraceGoal::key()
197+
{
198+
/* "a$" ensures substitution goals happen before derivation
199+
goals. */
200+
return "a$" + std::string(id.to_string(worker.store));
201+
}
202+
203+
void BuildTraceGoal::handleEOF(Descriptor fd)
204+
{
205+
worker.wakeUp(shared_from_this());
206+
}
207+
208+
} // namespace nix

‎src/libstore/build/derivation-goal.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ std::string DerivationGoal::key()
5252
i.e. a derivation named "aardvark" always comes before
5353
"baboon". And substitution goals always happen before
5454
derivation goals (due to "b$"). */
55-
return "b$" + std::string(drvPath.name()) + "$" + SingleDerivedPath::Built{
55+
return "c$" + std::string(drvPath.name()) + "$" + SingleDerivedPath::Built{
5656
.drvPath = makeConstantStorePathRef(drvPath),
5757
.output = wantedOutput,
5858
}.to_string(worker.store);

‎src/libstore/build/derivation-resolution-goal.cc‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#include "nix/store/build/derivation-resolution-goal.hh"
2+
#include "nix/util/finally.hh"
3+
#include "nix/store/build/worker.hh"
4+
#include "nix/store/build/substitution-goal.hh"
5+
#include "nix/util/callback.hh"
6+
#include "nix/store/derivations.hh"
7+
8+
namespace nix {
9+
10+
DerivationResolutionGoal::DerivationResolutionGoal(const StorePath & drvPath, Worker & worker)
11+
: Goal(worker, init())
12+
, drvPath(drvPath)
13+
{
14+
name = fmt("resolution of '%s'", worker.store.printStorePath(drvPath));
15+
trace("created");
16+
}
17+
18+
Goal::Co DerivationResolutionGoal::init()
19+
{
20+
trace("init");
21+
22+
std::unique_ptr<Derivation> drv;
23+
24+
if (worker.evalStore.isValidPath(drvPath)) {
25+
drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
26+
} else if (worker.store.isValidPath(drvPath)) {
27+
drv = std::make_unique<Derivation>(worker.store.readDerivation(drvPath));
28+
} else {
29+
auto goal0 = worker.makePathSubstitutionGoal(drvPath);
30+
goal0->preserveException = true;
31+
co_await await(Goals{goal0});
32+
if (nrFailed > 0)
33+
co_return amDone(goal0->exitCode, goal0->ex);
34+
35+
drv = std::make_unique<Derivation>(worker.store.readDerivation(drvPath));
36+
}
37+
38+
trace("output path substituted");
39+
40+
std::set<std::shared_ptr<BuildTraceGoal>> goals;
41+
42+
std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
43+
44+
accumInputPaths = [&](ref<SingleDerivedPath> depDrvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
45+
for (auto & outputName : inputNode.value)
46+
goals.insert(worker.makeBuildTraceGoal(SingleDerivedPath::Built{depDrvPath, outputName}));
47+
48+
for (auto & [outputName, childNode] : inputNode.childMap)
49+
accumInputPaths(make_ref<SingleDerivedPath>(SingleDerivedPath::Built{depDrvPath, outputName}), childNode);
50+
};
51+
52+
for (auto & [depDrvPath, depNode] : drv->inputDrvs.map)
53+
accumInputPaths(makeConstantStorePathRef(depDrvPath), depNode);
54+
55+
if (nrFailed > 0) {
56+
debug("TODO message");
57+
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
58+
}
59+
60+
if (true /*auto d = drv.tryResolve(....)*/) {
61+
// resolvedDerivation = d.take();
62+
63+
trace("finished");
64+
co_return amDone(ecSuccess);
65+
} else {
66+
// fail
67+
}
68+
}
69+
70+
std::string DerivationResolutionGoal::key()
71+
{
72+
/* "a$" ensures substitution goals happen before derivation
73+
goals. */
74+
return "b$" + worker.store.printStorePath(drvPath);
75+
}
76+
77+
void DerivationResolutionGoal::handleEOF(Descriptor fd)
78+
{
79+
worker.wakeUp(shared_from_this());
80+
}
81+
82+
} // namespace nix

0 commit comments

Comments
 (0)