Skip to content

Commit 02a1ea2

Browse files
committed
Add setup images for documentation
1 parent 21e9186 commit 02a1ea2

File tree

6 files changed

+196
-17
lines changed

6 files changed

+196
-17
lines changed

Examples/template.jl

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
### A Pluto.jl notebook ###
2-
# v0.20.6
2+
# v0.20.8
33

44
using Markdown
55
using InteractiveUtils
@@ -101,7 +101,6 @@ PlutoProfile = "~0.4.0"
101101
PlutoUI = "~0.7.61"
102102
ProgressLogging = "~0.1.4"
103103
StaticArrays = "~1.9.13"
104-
Statistics = "~1.11.1"
105104
StatsBase = "~0.34.4"
106105
Transducers = "~0.4.84"
107106
"""
@@ -110,9 +109,9 @@ Transducers = "~0.4.84"
110109
PLUTO_MANIFEST_TOML_CONTENTS = """
111110
# This file is machine-generated - editing it directly is not advised
112111
113-
julia_version = "1.11.4"
112+
julia_version = "1.11.5"
114113
manifest_format = "2.0"
115-
project_hash = "967a2aa9697bf04d488702953f3453db5cc9b108"
114+
project_hash = "fb175071a7c85444e2ee3da77dd1e4c18a1bb109"
116115
117116
[[deps.AbstractPlutoDingetjes]]
118117
deps = ["Pkg"]
@@ -586,7 +585,7 @@ version = "0.3.27+1"
586585
[[deps.OpenLibm_jll]]
587586
deps = ["Artifacts", "Libdl"]
588587
uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
589-
version = "0.8.1+4"
588+
version = "0.8.5+0"
590589
591590
[[deps.OpenSpecFun_jll]]
592591
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"]

NonTabularRL.jl/src/NonTabularRL.jl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import PlutoDevMacros
66

77
@reexport using ApproximationUtils
88

9-
include(joinpath(@__DIR__, "..", "..", "Chapter-9", "Chapter_9_On-policy_Prediction_with_Approximation.jl"))
9+
include(joinpath(@__DIR__, "..", "..", "Chapter-09", "Chapter_09_On-policy_Prediction_with_Approximation.jl"))
1010

1111
export gradient_monte_carlo_episode_update!, gradient_monte_carlo_policy_estimation!, gradient_monte_carlo_estimation!, semi_gradient_td0_update!, semi_gradient_td0_policy_estimation!, semi_gradient_td0_estimation!, semi_gradient_td0_policy_estimation, semi_gradient_td0_estimation, make_random_walk_mrp, state_aggregation_gradient_setup, run_state_aggregation_monte_carlo_policy_estimation, run_state_aggregation_semi_gradient_policy_estimation, order_features_gradient_setup, run_order_features_monte_carlo_policy_estimation, calc_poly_feature, calc_fourier_feature, tile_coding_gradient_setup, run_tile_coding_monte_carlo_policy_estimation
1212

TabularRL.jl/examples/defining_tabular_mdps.jl

Lines changed: 191 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,21 @@ using PlutoDevMacros, LinearAlgebra, Random, Statistics
1010
# ╔═╡ 98c33627-bd5e-4d9e-b3c0-0c215b3abc1c
1111
PlutoDevMacros.@frompackage @raw_str(joinpath(@__DIR__, "..", "..", "TabularRL.jl")) using TabularRL
1212

13+
# ╔═╡ 9a3df08c-7bc5-4508-a48a-44e01fcbbf5c
14+
begin
15+
using PlutoUI
16+
TableOfContents()
17+
end
18+
1319
# ╔═╡ 1504a12d-940e-4c03-9b83-6301b303d64b
1420
md"""
1521
# Installation
1622
17-
Once you have Julia [installed on your computer](https://julialang.org/install/) you can access the package management system from the REPL by typing `]`. From there you can install `TabularRL.jl` with the command: `add https://github.com/jekyllstein/Reinforcement-Learning-Sutton-Barto-Exercise-Solutions\TabularRL.jl`. If you do not want to add this package to your main environment, then activate a temporary one and add the package there. Below is an example of installing the package into a temporary environment from the REPL
23+
Both methods require you to [install julia](https://julialang.org/install/) on your computer. Julia should then be accessible in your shell environment with the commmand `julia`. Using this command will open the REPL, an example of which is shown in the first method below.
24+
25+
## Manual Setup in the REPL
26+
27+
After opening the REPL, you can access the package management system by typing `]`. From there you can install `TabularRL.jl` with the command: `add https://github.com/jekyllstein/Reinforcement-Learning-Sutton-Barto-Exercise-Solutions:TabularRL.jl`. If you do not want to add this package to your main environment, then activate a temporary one and add the package there. Below is an example of installing the package into a temporary environment from the REPL
1828
1929
```julia
2030
_ _ _(_)_ | Documentation: https://docs.julialang.org
@@ -27,19 +37,83 @@ Once you have Julia [installed on your computer](https://julialang.org/install/)
2737
2838
julia>]
2939
(@v1.11) pkg> activate --temp
30-
Activating new project at `/tmp/jl_CE8nWh`
31-
(jl_C#8nWh) pkg> add https://github.com/jekyllstein/Reinforcement-Learning-Sutton-Barto-Exercise-Solutions/TabularRL.jl
40+
Activating new project at `/tmp/jl_w0iM0P`
41+
(jl_w0iM0P) pkg> add https://github.com/jekyllstein/Reinforcement-Learning-Sutton-Barto-Exercise-Solutions:TabularRL.jl
42+
Updating git-repo `https://github.com/jekyllstein/Reinforcement-Learning-Sutton-Barto-Exercise-Solutions`
43+
Resolving package versions...
44+
Updating `/tmp/jl_w0iM0P/Project.toml`
45+
[70984187] + TabularRL v0.1.0 `https://github.com/jekyllstein/Reinforcement-Learning-Sutton-Barto-Exercise-Solutions#jolin_dev_dev:TabularRL.jl`
46+
Updating `/tmp/jl_w0iM0P/Manifest.toml`
47+
[7d9f7c33] + Accessors v0.1.42
48+
.
49+
.
50+
.
51+
3252
```
3353
"""
3454

35-
# ╔═╡ d3db6af7-a365-47fa-965c-e7057938468e
36-
dump(TabularMDP)
55+
# ╔═╡ 2d0a1689-93d6-4be8-a991-64a2f57f8475
56+
md"""
57+
After the package is installed you can return to normal REPL mode by hitting `backspace` and then enter the command `using TabularRL`
58+
"""
3759

38-
# ╔═╡ ed741cb4-32c1-4287-8368-6835e94d3496
60+
# ╔═╡ 12d83219-d470-40e9-8bee-a3db0b2bd2a8
3961
md"""
4062
```julia
41-
julia> states = 1:10
63+
julia> using TabularRL
64+
```
65+
"""
66+
67+
# ╔═╡ 761318f2-095b-4e3f-a320-061e9f50f166
68+
md"""
69+
## Automatic Setup in Notebooks
70+
71+
Alternatively, you can clone the entire reinforcement learning exercise repository and have access to every notebook and package contained therein. Check to see if you have `git` installed on your computer with `git --version`. If you receive an error message or do not see a version number then [install git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git).
72+
73+
Clone the repository to your system inside a directory where you have read/write access:
74+
75+
```shell
76+
> git clone https://github.com/jekyllstein/Reinforcement-Learning-Sutton-Barto-Exercise-Solutions.git
77+
```
78+
79+
Navigate inside of the repository directory where you should find the following shell scripts: `setup.sh`, `start.sh`, `update.sh`. Note that you may need to make these files executable with
80+
81+
```shell
82+
> chmod +x setup.sh```
83+
```
84+
85+
You can begin by running the start shell script which will automatically run the setup script if this is your first time using the repository:
86+
87+
```shell
88+
> ./start.sh
89+
```
90+
91+
After some precompilation and setup, you should see the following at the bottom of the terminal:
92+
93+
```julia
94+
[ Info: Loading...
95+
┌ Info:
96+
└ Go to http://localhost:1234/?secret=3Ah66MkG in your browser to start writing ~ have fun!
97+
┌ Info:
98+
│ Press Ctrl+C in this terminal to stop Pluto
4299
```
100+
101+
However, note that the secret and port number may differ on your system. The URL that contains `localhost` is what you should copy into your web browser to see the Pluto welcome screen.
102+
"""
103+
104+
# ╔═╡ 1519dfbc-e593-4f1e-9b09-9af8157b04b8
105+
md"""
106+
![](pluto_welcome.png)
107+
"""
108+
109+
# ╔═╡ 418687b7-73f4-476d-8eeb-9791830f44e3
110+
md"""
111+
If you click in the text box under `Open a notebook` a navigation menu will appear that shows the directory structure. If you open any of the `Chapter...` folders, you will see notebook files which can be opened and used interactively. For our purposes, however, we will open a template notebook which loads all of the required tools. This notebook is contained at `Examples/template.jl` and can be opened from the text box (see below).
112+
"""
113+
114+
# ╔═╡ 20ebb844-bde2-41fd-a512-d62991e2f6d0
115+
md"""
116+
By default, the notebook will open in a preview mode (see below). Click `Run notebook code` at the top to run the notebook and have access to all the tools. From there you can add cells to the notebook and enter commands in them just like you would in the REPL. The code examples which follow can work either in the REPL or the notebook.
43117
"""
44118

45119
# ╔═╡ b328f330-368c-11f0-107d-4b2801866e56
@@ -52,11 +126,13 @@ PLUTO_PROJECT_TOML_CONTENTS = """
52126
[deps]
53127
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
54128
PlutoDevMacros = "a0499f29-c39b-4c5c-807c-88074221b949"
129+
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
55130
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
56131
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
57132
58133
[compat]
59134
PlutoDevMacros = "~0.9.0"
135+
PlutoUI = "~0.7.62"
60136
"""
61137

62138
# ╔═╡ 00000000-0000-0000-0000-000000000002
@@ -65,7 +141,13 @@ PLUTO_MANIFEST_TOML_CONTENTS = """
65141
66142
julia_version = "1.11.5"
67143
manifest_format = "2.0"
68-
project_hash = "b839a1d69d57d65c7371c862085f0a8f9ad7fa5b"
144+
project_hash = "f87f53f7371c1d9d6afcb48a3d9214024f41503e"
145+
146+
[[deps.AbstractPlutoDingetjes]]
147+
deps = ["Pkg"]
148+
git-tree-sha1 = "6e1d2a35f2f90a4bc7c2ed98079b2ba09c35b83a"
149+
uuid = "6e696c72-6542-2067-7265-42206c756150"
150+
version = "1.3.2"
69151
70152
[[deps.ArgTools]]
71153
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
@@ -85,6 +167,12 @@ git-tree-sha1 = "062c5e1a5bf6ada13db96a4ae4749a4c2234f521"
85167
uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
86168
version = "1.3.9"
87169
170+
[[deps.ColorTypes]]
171+
deps = ["FixedPointNumbers", "Random"]
172+
git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d"
173+
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
174+
version = "0.11.5"
175+
88176
[[deps.CompilerSupportLibraries_jll]]
89177
deps = ["Artifacts", "Libdl"]
90178
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
@@ -104,11 +192,41 @@ version = "1.6.0"
104192
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
105193
version = "1.11.0"
106194
195+
[[deps.FixedPointNumbers]]
196+
deps = ["Statistics"]
197+
git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172"
198+
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
199+
version = "0.8.5"
200+
201+
[[deps.Hyperscript]]
202+
deps = ["Test"]
203+
git-tree-sha1 = "179267cfa5e712760cd43dcae385d7ea90cc25a4"
204+
uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91"
205+
version = "0.0.5"
206+
207+
[[deps.HypertextLiteral]]
208+
deps = ["Tricks"]
209+
git-tree-sha1 = "7134810b1afce04bbc1045ca1985fbe81ce17653"
210+
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
211+
version = "0.9.5"
212+
213+
[[deps.IOCapture]]
214+
deps = ["Logging", "Random"]
215+
git-tree-sha1 = "b6d6bfdd7ce25b0f9b2f6b3dd56b2673a66c8770"
216+
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
217+
version = "0.2.5"
218+
107219
[[deps.InteractiveUtils]]
108220
deps = ["Markdown"]
109221
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
110222
version = "1.11.0"
111223
224+
[[deps.JSON]]
225+
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
226+
git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a"
227+
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
228+
version = "0.21.4"
229+
112230
[[deps.JuliaInterpreter]]
113231
deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"]
114232
git-tree-sha1 = "c47892541d03e5dc63467f8964c9f2b415dfe718"
@@ -153,6 +271,11 @@ version = "1.11.0"
153271
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
154272
version = "1.11.0"
155273
274+
[[deps.MIMEs]]
275+
git-tree-sha1 = "c64d943587f7187e751162b3b84445bbbd79f691"
276+
uuid = "6c6e2e6c-3030-632d-7369-2d6c69616d65"
277+
version = "1.1.0"
278+
156279
[[deps.MacroTools]]
157280
git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522"
158281
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
@@ -168,6 +291,10 @@ deps = ["Artifacts", "Libdl"]
168291
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
169292
version = "2.28.6+0"
170293
294+
[[deps.Mmap]]
295+
uuid = "a63ad114-7e13-5084-954f-fe012c677804"
296+
version = "1.11.0"
297+
171298
[[deps.MozillaCACerts_jll]]
172299
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
173300
version = "2023.12.12"
@@ -181,6 +308,12 @@ deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
181308
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
182309
version = "0.3.27+1"
183310
311+
[[deps.Parsers]]
312+
deps = ["Dates", "PrecompileTools", "UUIDs"]
313+
git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810"
314+
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
315+
version = "2.8.3"
316+
184317
[[deps.Pkg]]
185318
deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"]
186319
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -198,6 +331,24 @@ git-tree-sha1 = "72f65885168722413c7b9a9debc504c7e7df7709"
198331
uuid = "a0499f29-c39b-4c5c-807c-88074221b949"
199332
version = "0.9.0"
200333
334+
[[deps.PlutoUI]]
335+
deps = ["AbstractPlutoDingetjes", "Base64", "ColorTypes", "Dates", "FixedPointNumbers", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "MIMEs", "Markdown", "Random", "Reexport", "URIs", "UUIDs"]
336+
git-tree-sha1 = "d3de2694b52a01ce61a036f18ea9c0f61c4a9230"
337+
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
338+
version = "0.7.62"
339+
340+
[[deps.PrecompileTools]]
341+
deps = ["Preferences"]
342+
git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
343+
uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
344+
version = "1.2.1"
345+
346+
[[deps.Preferences]]
347+
deps = ["TOML"]
348+
git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
349+
uuid = "21216c6a-2e73-6563-6e65-726566657250"
350+
version = "1.4.3"
351+
201352
[[deps.Printf]]
202353
deps = ["Unicode"]
203354
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
@@ -208,10 +359,19 @@ deps = ["SHA"]
208359
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
209360
version = "1.11.0"
210361
362+
[[deps.Reexport]]
363+
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
364+
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
365+
version = "1.2.2"
366+
211367
[[deps.SHA]]
212368
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
213369
version = "0.7.0"
214370
371+
[[deps.Serialization]]
372+
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
373+
version = "1.11.0"
374+
215375
[[deps.Statistics]]
216376
deps = ["LinearAlgebra"]
217377
git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0"
@@ -234,6 +394,21 @@ deps = ["ArgTools", "SHA"]
234394
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
235395
version = "1.10.0"
236396
397+
[[deps.Test]]
398+
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
399+
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
400+
version = "1.11.0"
401+
402+
[[deps.Tricks]]
403+
git-tree-sha1 = "6cae795a5a9313bbb4f60683f7263318fc7d1505"
404+
uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775"
405+
version = "0.1.10"
406+
407+
[[deps.URIs]]
408+
git-tree-sha1 = "cbbebadbcc76c5ca1cc4b4f3b0614b3e603b5000"
409+
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
410+
version = "1.5.2"
411+
237412
[[deps.UUIDs]]
238413
deps = ["Random", "SHA"]
239414
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -265,11 +440,16 @@ version = "17.4.0+2"
265440
"""
266441

267442
# ╔═╡ Cell order:
268-
# ╠═1504a12d-940e-4c03-9b83-6301b303d64b
269-
# ╠═d3db6af7-a365-47fa-965c-e7057938468e
270-
# ╠═ed741cb4-32c1-4287-8368-6835e94d3496
443+
# ╟─1504a12d-940e-4c03-9b83-6301b303d64b
444+
# ╟─2d0a1689-93d6-4be8-a991-64a2f57f8475
445+
# ╟─12d83219-d470-40e9-8bee-a3db0b2bd2a8
446+
# ╟─761318f2-095b-4e3f-a320-061e9f50f166
447+
# ╠═1519dfbc-e593-4f1e-9b09-9af8157b04b8
448+
# ╟─418687b7-73f4-476d-8eeb-9791830f44e3
449+
# ╟─20ebb844-bde2-41fd-a512-d62991e2f6d0
271450
# ╟─b328f330-368c-11f0-107d-4b2801866e56
272451
# ╠═88fc8fef-c10b-408c-82d5-cb97c87aa363
273452
# ╠═98c33627-bd5e-4d9e-b3c0-0c215b3abc1c
453+
# ╠═9a3df08c-7bc5-4508-a48a-44e01fcbbf5c
274454
# ╟─00000000-0000-0000-0000-000000000001
275455
# ╟─00000000-0000-0000-0000-000000000002
568 KB
Loading
173 KB
Loading
12.6 KB
Loading

0 commit comments

Comments
 (0)