Try to find out the wrong in the test

本文介绍了一道关于分组问题的算法题,通过分治与线段树的巧妙结合来解决。问题中每个人有特定的人数限制,需要最大化分组数量。文章详细解析了解决方案,并给出了具体实现代码。

Description

有n个人排成一排,你需要对这n个人分组,每组必须是连续的一段。
每个人有要求,(c[i],d[i])表示这个人所在的组的最少人数和最多人数。
求最多能分成多少组和方案。
n<=1e6

Solution

如果只有d的限制这道题就很好做了。
因为d限制了我们i只能从i前面的一段区间转移过来,不妨设为left[i],显然left是单调的
但是有c的限制就很麻烦了,因为c的限制是一堆零碎的区间。
于是我们可以考虑按c的最大值分治。设c的最大值的位置为k,值为c。
我们先处理[l,k-1],然后考虑跨过k的转移。
这样子我们的下界就一定是k。
接下来我们考虑分类讨论
1:left[i] < l&&i<=k-1+c
这样子的点的转移区间是左边的一段区间,并且随着i的移动可转移的区间也不断扩大。
于是我们可以对第一个位置用线段树查出它的答案,接下来线性更新即可。
2:left[i ]< l&&i>k-1+c
符合条件的i也一定是一段区间,并且转移区间是整个左区间,于是直接用线段树维护就好了。
3:l<=left[i] < k
直接在线段树上查找。
4:left[i]>=k
分治到[k,r]区间。

但是我们这样不是常规的分治,如何考虑复杂度?
第1种情况最开始显然是O(n log n)的
考虑i的移动,最坏情况下i会移动min(k-l,r-k+1)次,也就是两边的长度取min
把分治过程看成启发式合并,这一部分也是O(n log n)的
第2种情况显然是O(n log n)的
第3种情况,对于每个点我们只会做一次,所以也是O(n log n)的
至此我们已经完美的解决了这道神题Orz

Code

#include <cstdio>
#include <cstring>
#include <algorithm>
#define fo(i,a,b) for(int i=a;i<=b;i++)
#define fd(i,a,b) for(int i=a;i>=b;i--)
using namespace std;

int read() {
    char ch;
    for(ch=getchar();ch<'0'||ch>'9';ch=getchar());
    int x=ch-'0';
    for(ch=getchar();ch>='0'&&ch<='9';ch=getchar()) x=x*10+ch-'0';
    return x;
}

const int N=1e6+5,Mo=1e9+7;

#define max(a,b) ((a)>(b)?(a):(b))
#define min(a,b) ((a)<(b)?(a):(b))

struct node{
    int mx,cnt;
    node(int _mx=0,int _cnt=0) {mx=_mx;cnt=_cnt;}
    friend node operator + (node y,node z) {
        if (y.mx==-1) return z;
        if (z.mx==-1) return y;
        node x;x.mx=max(y.mx,z.mx);
        if (y.mx==z.mx) x.cnt=(y.cnt+z.cnt)%Mo;
        if (y.mx>z.mx) x.cnt=y.cnt;
        if (y.mx<z.mx) x.cnt=z.cnt;
        return x;
    }
}tr[N<<2],lazy[N<<2],f[N];

int n,mx[N<<2],c[N],d[N],q[N],left[N];

void update(int v,node z) {
    tr[v]=tr[v]+z;lazy[v]=lazy[v]+z;
}

void down(int v) {
    if (lazy[v].mx!=-1) {
        update(v<<1,lazy[v]);
        update(v<<1|1,lazy[v]);
        lazy[v]=node(-1,0);
    }
}

void build(int v,int l,int r) {
    if (l==r) {mx[v]=l;return;}
    int mid=l+r>>1;
    build(v<<1,l,mid);
    build(v<<1|1,mid+1,r);
    if (c[mx[v<<1]]>=c[mx[v<<1|1]]) mx[v]=mx[v<<1];
    else mx[v]=mx[v<<1|1];
}

int get_max(int v,int l,int r,int x,int y) {
    if (l==x&&r==y) return mx[v];
    int mid=l+r>>1;
    if (y<=mid) return get_max(v<<1,l,mid,x,y);
    else if (x>mid) return get_max(v<<1|1,mid+1,r,x,y);
    else {
        int a=get_max(v<<1,l,mid,x,mid);
        int b=get_max(v<<1|1,mid+1,r,mid+1,y);
        return (c[a]>=c[b])?a:b;
    }
}

void modify(int v,int l,int r,int x,int y,node z) {
    if (x>y) return;
    if (l==x&&r==y) {update(v,z);return;}
    int mid=l+r>>1;down(v);
    if (y<=mid) modify(v<<1,l,mid,x,y,z);
    else if (x>mid) modify(v<<1|1,mid+1,r,x,y,z);
    else modify(v<<1,l,mid,x,mid,z),modify(v<<1|1,mid+1,r,mid+1,y,z);
    tr[v]=tr[v<<1]+tr[v<<1|1];
}

node query(int v,int l,int r,int x,int y) {
    if (x>y) return node(-1,0);
    if (l==x&&r==y) return tr[v];
    int mid=l+r>>1;down(v);
    if (y<=mid) return query(v<<1,l,mid,x,y);
    else if (x>mid) return query(v<<1|1,mid+1,r,x,y);
    else return query(v<<1,l,mid,x,mid)+query(v<<1|1,mid+1,r,mid+1,y);
}

void init() {
    n=read();
    fo(i,1,n) c[i]=read(),d[i]=read();
    int head=1,tail=0;
    fo(i,1,n) {
        while (head<=tail&&d[q[tail]]>d[i]) tail--;
        q[++tail]=i;
        for(int j=left[i-1];;j++) {
            while (head<=tail&&q[head]<=j) head++;
            if (i-j<=d[q[head]]) {left[i]=j;break;}
        }
    }
    fo(i,0,n) f[i]=node(-1,0);f[0]=node(0,1);
    fo(i,1,n<<2) tr[i]=lazy[i]=node(-1,0);
}

int find(int l,int r,int v) {
    r++;
    while (l<r) {
        int mid=l+r>>1;
        if (left[mid]<v) l=mid+1;
        else r=mid;
    }
    return l-1;
}

void solve(int l,int r) {
    if (l>r) return;
    if (l==r) {
        modify(1,0,n,l,l,f[l]);
        f[l]=query(1,0,n,l,l);
        return;
    }
    int k=get_max(1,0,n,l+1,r),mx=c[k];
    solve(l,k-1);
    int mid=max(l+mx,k);
    if (left[mid]<l&&mid<=r) {
        node now=query(1,0,n,l,mid-mx);
        if (now.mx!=-1) {
            now.mx++;
            f[mid]=f[mid]+now;
            now.mx--;
        }
        fo(i,mid+1,min(k-1+mx,r)) {
            if (left[i]>=l) break;
            now=now+f[i-mx];
            if (now.mx!=-1) {
                now.mx++;
                f[i]=f[i]+now;
                now.mx--;
            }
        }
    }

    mid=k+mx;
    if (left[mid]<l&&mid<=r) {
        node now=query(1,0,n,l,k-1);
        if (now.mx!=-1) {
            now.mx++;
            modify(1,0,n,mid,find(mid,r,l),now);
        }
    }

    mid=find(k,r,l)+1;
    fo(i,mid,r) {
        if (left[i]>=k) break;
        node now=query(1,0,n,left[i],min(k-1,i-mx));
        if (now.mx!=-1) {
            now.mx++;
            f[i]=f[i]+now;
        }
    }
    solve(k,r);
}

int main() {
    freopen("schooldays.in","r",stdin);
    freopen("schooldays.out","w",stdout);
    init();
    build(1,0,n);
    solve(0,n);
    if (f[n].mx==-1) puts("-1");
    else printf("%d %d\n",f[n].mx,f[n].cnt);
    return 0;
}
请翻译下面内容为中文, ====================================== INSTALLING SUBVERSION A Quick Guide ====================================== $LastChangedDate$ Contents: I. INTRODUCTION A. Audience B. Dependency Overview C. Dependencies in Detail D. Documentation II. INSTALLATION A. Building from a Tarball B. Building the Latest Source under Unix C. Building under Unix in Different Directories D. Installing from a Zip or Installer File under Windows E. Building the Latest Source under Windows F. Building using CMake III. BUILDING A SUBVERSION SERVER A. Setting Up Apache Httpd B. Making and Installing the Subversion Apache Server Module C. Configuring Apache Httpd for Subversion D. Running and Testing E. Alternative: 'svnserve' and ra_svn IV. PROGRAMMING LANGUAGE BINDINGS (PYTHON, PERL, RUBY, JAVA) I. INTRODUCTION ============ A. Audience This document is written for people who intend to build Subversion from source code. Normally, the only people who do this are Subversion developers and package maintainers. If neither of these labels fits you, we recommend you find an appropriate binary package of Subversion and install that. While the Subversion project doesn't officially release binary packages, a number of volunteers have made such packages available for different operating systems. Most Linux and BSD distributions already have Subversion packages ready to go via standard packaging channels, and other volunteers have built 'installers' for both Windows and OS X. Visit this page for package links: https://subversion.apache.org/packages.html For those of you who still wish to build from source, Subversion follows the Unix convention of "./configure && make", but it has a number of dependencies. B. Dependency Overview You'll need the following build tools to compile Subversion: * autoconf 2.59 or later (Unix only) * libtool 1.4 or later (Unix only) * a reasonable C compiler (gcc, Visual Studio, etc.) Subversion also depends on the following third-party libraries: * libapr and libapr-util (REQUIRED for client and server) The Apache Portable Runtime (APR) library provides an abstraction of operating-system level services such as file and network I/O, memory management, and so on. It also provides convenience routines for things like hashtables, checksums, and argument processing. While it was originally developed for the Apache HTTP server, APR is a standalone library used by Subversion and other products. It is a critical dependency for all of Subversion; it's the layer that allows Subversion clients and servers to run on different operating systems. * SQLite (REQUIRED for client and server) Subversion uses SQLite to manage some internal databases. * libz (REQUIRED for client and server) Subversion uses zlib for compressing binary differences. These diff streams are used everywhere -- over the network, in the repository, and in the client's working copy. * utf8proc (REQUIRED for client and server) Subversion uses utf8proc for UTF-8 support, including Unicode normalization. * Apache Serf (OPTIONAL for client) The Apache Serf library allows the Subversion client to send HTTP requests. This is necessary if you want your client to access a repository served by the Apache HTTP server. There is an alternate 'svnserve' server as well, though, and clients automatically know how to speak the svnserve protocol. Thus it's not strictly necessary for your client to be able to speak HTTP... though we still recommend that your client be built to speak both HTTP and svnserve protocols. * OpenSSL (OPTIONAL for client and server) OpenSSL enables your client to access SSL-encrypted https:// URLs (using Apache Serf) in addition to unencrypted http:// URLs. To use SSL with Subversion's WebDAV server, Apache needs to be compiled with OpenSSL as well. * Netwide Assembler (OPTIONAL for client and server) The Netwide Assembler (NASM) is used to build the (optional) assembler modules of OpenSSL. As of OpenSSL 1.1.0 NASM is the only supported assembler. * Berkeley DB (DEPRECATED and OPTIONAL for client and server) When you create a repository, you have the option of specifying a storage 'back-end' implementation. Currently, there are two options. The newer and recommended one, known as FSFS, does not require Berkeley DB. FSFS stores data in a flat filesystem. The older implementation, known as BDB, has been deprecated and is not recommended for new repositories, but is still available. BDB stores data in a Berkeley DB database. This back-end will only be available if the BDB libraries are discovered at compile time. * libsasl (OPTIONAL for client and server) If the Cyrus SASL library is detected at compile time, then the svn client (and svnserve server) will be able to utilize SASL to do various forms of authentication when speaking the svnserve protocol. * Python, Perl, Java, Ruby (OPTIONAL) Subversion is mostly a collection of C libraries with well-defined APIs, with a small collection of programs that use the APIs. If you want to build Subversion API bindings for other languages, you need to have those languages available at build time. * py3c (OPTIONAL, but REQUIRED for Python bindings) The Python 3 Compatibility Layer for C Extensions is required to build the Python language bindings. * KDE Framework 5, libsecret, GNOME Keyring (OPTIONAL for client) Subversion contains optional support for storing passwords in KWallet via KDE Framework 5 libraries (preferred) or kdelibs4, and GNOME Keyring via libsecret (preferred) or GNOME APIs. * libmagic (OPTIONAL) If the libmagic library is detected at compile time, it will be used to determine mime-types of binary files which are added to version control. Note that mime-types configured via auto-props or the mime-types-file option take precedence. C. Dependencies in Detail Subversion depends on a number of third party tools and libraries. Some of them are only required to run a Subversion server; others are necessary just for a Subversion client. This section explains what other tools and libraries will be required so that Subversion can be built with the set of features you want. On Unix systems, the './configure' script will tell you if you are missing the correct version of any of the required libraries or tools, so if you are in a real hurry to get building, you can skip straight to section II. If you want to gather the pieces you will need before starting out, however, you should read the following. If you're just installing a Subversion client, the Subversion team has created a script that downloads the minimal prerequisite libraries (Apache Portable Runtime, Sqlite, and Zlib). The script, 'get-deps.sh', is available in the same directory as this file. When run, it will place 'apr', 'apr-util', 'serf', 'zlib', and 'sqlite-amalgamation' directories directly into your unpacked Subversion distribution. With the exception of sqlite-amalgamation, they will still need to be configured, built and installed explicitly, and Subversion's own configure script may need to be told where to find them, if they were not installed in standard system locations. Note: there are optional dependencies (such as OpenSSL, swig, and httpd) which get-deps.sh does not download. Note: Because previous builds of Subversion may have installed older versions of these libraries, you may want to run some of the cleanup commands described in section II.B before installing the following. 1. Apache Portable Runtime 1.4 or newer (REQUIRED) Whenever you want to build any part of Subversion, you need the Apache Portable Runtime (APR) and the APR Utility (APR-util) libraries. If you do not have a pre-installed APR and APR-util, you will need to get these yourself: https://apr.apache.org/download.cgi On Unix systems, if you already have the APR libraries compiled and do not wish to regenerate them from source code, then Subversion needs to be able to find them. There are a couple of options to "./configure" that tell it where to look for the APR and APR-util libraries. By default it will try to locate the libraries using apr-config and apu-config scripts. These scripts provide all the relevant information for the APR and APR-util installations. If you want to specify the location of the APR library, you can use the "--with-apr=" option of "./configure". It should be able to find the apr-config script in the standard location under that directory (e.g. ${prefix}/bin). Similarly, you can specify the location of APR-util using the "--with-apr-util=" option to "./configure". It will look for the apu-config script relative to that directory. For example, if you want to use the APR libraries you built with the Apache httpd server, you could run: $ ./configure --with-apr=/usr/local/apache2 \ --with-apr-util=/usr/local/apache2 ... Notes on Windows platforms: * Do not use APR version 1.7.3 as that release contains a bug that makes it impossible for Subversion to use it properly. This issue only affects APR builds on Windows. This issue was fixed in APR version 1.7.4. See: https://lists.apache.org/thread/xd5t922jvb9423ph4j84rsp5fxks1k0z * If you check out APR and APR-util sources from their Subversion repository, be sure to use a native Windows SVN client (as opposed to Cygwin's version) so that the .dsp files get carriage-returns at the ends of their lines. Otherwise Visual Studio will complain that it doesn't recognize the .dsp files. Notes on Unix platforms: * If you check out APR and APR-util sources from their Subversion repository, you need to run the 'buildconf' script in each library's directory to regenerate the configure scripts and other files required for compiling the libraries. Afterwards, configure, build, and install both libraries before running Subversion's configure script. For example: $ cd apr $ ./buildconf $ ./configure <options...> $ make $ make install $ cd .. $ cd apr-util $ ./buildconf $ ./configure <options...> $ make $ make install $ cd .. 2. SQLite (REQUIRED) Subversion requires SQLite version 3.24.0 or above. You can meet this dependency several ways: * Use an SQLite amalgamation file. * Specify an SQLite installation to use. * Let Subversion find an installed SQLite. To use an SQLite-provided amalgamation, just drop sqlite3.c into Subversion's sqlite-amalgamation/ directory, or point to it with the --with-sqlite configure option. This file also ships with the Subversion dependencies distribution, or you can download it from SQLite: https://www.sqlite.org/download.html 3. Zlib (REQUIRED) Subversion's binary-differencing engine depends on zlib for compression. Most Unix systems have libz pre-installed, but if you need it, you can get it from http://www.zlib.net/ 4. utf8proc (REQUIRED) Subversion uses utf8proc for UTF-8 support. Configure will attempt to locate utf8proc by default using pkg-config and known paths. If it is installed in a non-standard location, then use: --with-utf8proc=/path/to/libutf8proc Alternatively, a copy of utf8proc comes bundled with the Subversion sources. If configure should use the bundled copy, use: --with-utf8proc=internal 5. autoconf 2.59 or newer (Unix only) This is required only if you plan to build from the latest source (see section II.B). Generally only developers would be doing this. 6. libtool 1.4 or newer (Unix only) This is required only if you plan to build from the latest source (see section II.B). Note: Some systems (Solaris, for example) require libtool 1.4.3 or newer. The autogen.sh script knows about that. 7. Apache Serf library 1.3.4 or newer (OPTIONAL) If you want your client to be able to speak to an Apache server (via a http:// or https:// URL), you must link against Apache Serf. Though optional, we strongly recommend this. In order to use ra_serf, you must install serf, and run Subversion's ./configure with the argument --with-serf. If serf is installed in a non-standard place, you should use --with-serf=/path/to/serf/install instead. Apache Serf can be obtained via your system's package distribution system or directly from https://serf.apache.org/. For more information on Apache Serf and Subversion's ra_serf, see the file subversion/libsvn_ra_serf/README. 8. OpenSSL (OPTIONAL) ### needs some updates. I think Apache Serf automagically handles ### finding OpenSSL, but we may need more docco here. and w.r.t ### zlib. The Apache Serf library has support for SSL encryption by relying on the OpenSSL library. a. Using OpenSSL on the client through Apache Serf On Unix systems, to build Apache Serf with OpenSSL, you need OpenSSL installed on your system, and you must add "--with-ssl" as a "./configure" parameter. If your OpenSSL installation is hard for Apache Serf to find, you may need to use "--with-libs=/path/to/lib" in addition. In particular, on Red Hat (but not Fedora Core) it is necessary to specify "--with-libs=/usr/kerberos" for OpenSSL to be found. You can also specify a path to the zlib library using "--with-libs". Under Windows, you can specify the paths to these libraries by passing the options --with-zlib and --with-openssl to gen-make.py. b. Using OpenSSL on the Apache server You can also add support for these features to an Apache httpd server to be used for Subversion using the same support libraries. The Subversion build system will not provide them, however. You add them by specifying parameters to the "./configure" script of the Apache Server instead. For getting SSL on your server, you would add the "--enable-ssl" or "--with-ssl=/path/to/lib" option to Apache's "./configure" script. Apache enables zlib support by default, but you can specify a nonstandard location for the library with the "--with-z=/path/to/dir" option. Consult the Apache documentation for more details, and for other modules you may wish to install to enhance your Subversion server. If you don't already have it, you can get a copy of OpenSSL, including instructions for building and packaging on both Unix systems and Windows, at: https://www.openssl.org/ 9. Berkeley DB 4.X (DEPRECATED and OPTIONAL) You need the Berkeley DB libraries only if you are building a Subversion server that supports the older BDB repository storage back-end, or a Subversion client that can access local BDB repositories via the file:// URI scheme. The BDB back-end has been deprecated and is not recommended for new repositories. BDB may be removed in Subversion 2.0. We recommend the newer FSFS back-end for all new repositories. FSFS does not require the Berkeley DB libraries. If in doubt, the 'svnadmin info' command, added in Subversion 1.9, can identify whether an existing repository uses BDB or FSFS. The current recommended version of Berkeley DB is 4.4.20 or newer, which brings auto-recovery functionality to the Berkeley DB database environment. If you must use an older version of Berkeley DB, we *strongly* recommend using 4.3 or 4.2 over the 4.1 or 4.0 versions. Not only are these significantly faster and more stable, but they also enable Subversion repositories to automatically clean up database journal files to save disk space. You'll need Berkeley DB installed on your system. You can get it from: http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/overview/index.html If you have Berkeley DB installed in a place not searched by default for includes and libraries, add something like this: --with-berkeley-db=db.h:/usr/local/include/db4.7:/usr/local/lib/db4.7:db-4.7 to your `configure' switches, and the build process will use the Berkeley DB header and library in the named directories. You may need to use a different path, of course. Note that in order for the detection to succeed, the dynamic linker must be able to find the libraries at configure time. 10. Cyrus SASL library (OPTIONAL) If the Simple Authentication and Security Layer (SASL) library is detected on your system, then the Subversion client and svnserve server can utilize its abilities for various forms of authentication. To learn more about SASL or to get the source code, visit: http://freshmeat.net/projects/cyrussasl/ 11. Apache Web Server 2.2.X or newer (OPTIONAL) (https://httpd.apache.org/download.cgi) The Apache httpd server is one of two methods to make your Subversion repository available over a network - the other is a custom server program called svnserve, which requires no extra software packages. Building Subversion, the Apache server, and the modules that Apache needs to communicate with Subversion are complicated enough that there is a whole section at the end of this document that describes how it is done: See section III for details. 12. Python 3.x or newer (https://www.python.org/) (OPTIONAL) Subversion does not require Python for its basic operation. However, Python is required for building and testing Subversion and for using Subversion's SWIG Python bindings or hook scripts coded in Python. The majority of Subversion's test suite is written in Python, as is part of Subversion's build system. In more detail, Python is required to do any of the following: * Use the SWIG Python bindings. * Use the ctypes Python bindings. * Use hook scripts coded in Python. * Build Subversion from a tarball on Unix-like systems and run Subversion's test suite as described in section II.B. * Build Subversion on Windows as described in section II.E. * Build Subversion from a working copy checked out from Subversion's own repository (whether or not running the test suite). * Build the SWIG Python bindings. * Build the ctypes Python bindings. * Testing as described in section III.D. The Python bindings are used by: * Third-party programs (e.g., ViewVC) * Scripts distributed with Subversion itself in the tools/ subdirectory. * Any in-house scripts you may have. Python is NOT required to do any of the following: * Use the core command-line binaries (svn, svnadmin, svnsync, etc.) * Use Subversion's C libraries. * Use any of Subversion's other language bindings. * Build Subversion from a tarball on Unix-like systems without running Subversion's test suite Although this section calls for Python 3.x, Subversion still technically works with Python 2.7. However, Support for Python 2.7 is being phased out. As of 1 January 2020, Python 2.7 has reached end of life. All users are strongly encouraged to move to Python 3. Note: If you are using a Subversion distribution tarball and want to build the Python bindings for Python 2, you should rebuild the build environment in non-release mode by running 'sh autogen.sh' before running the ./configure script; see section II.B for more about autogen.sh. 13. Perl 5.8 or newer (Windows only) (OPTIONAL) To build Subversion under any of the MS Windows platforms, you will also need Perl 5.8 or newer to run apr-util's w32locatedb.pl script. 14. pkg-config (Unix only, OPTIONAL) Subversion uses pkg-config to find appropriate options used at build time. 15. D-Bus (Unix only, OPTIONAL) D-Bus is a message bus system. D-Bus is required for support for KWallet and GNOME Keyring. pkg-config is needed to find D-Bus headers and library. 16. Qt 5 or Qt 4 (Unix only, OPTIONAL) Qt is a cross-platform application framework. QtCore, QtDBus and QtGui modules are required for support for KWallet. pkg-config is needed to find Qt headers and libraries. 17. KDE 5 Framework libraries or KDELibs 4 (Unix only, OPTIONAL) Subversion contains optional support for storing passwords in KWallet. Subversion will look for KF5Wallet, KF5CoreAddons, KF5I18n APIs by default, and needs kf5-config to find them. The KDELibs 4 api is also supported. KDELibs contains core KDE libraries. Subversion uses libkdecore and libkdeui libraries when support for KWallet is enabled. kde4-config is used to get some necessary options. pkg-config, D-Bus and Qt 4 are also required. If you want to build support for KWallet, then pass the '--with-kwallet' option to `configure`. If KDE is installed in a non-standard prefix, then use: --with-kwallet=/path/to/KDE/prefix 18. GLib 2 (Unix only, OPTIONAL) GLib is a general-purpose utility library. GLib is required for support for GNOME Keyring. pkg-config is needed to find GLib headers and library. 19. GNOME Keyring (Unix only, OPTIONAL) Subversion contains optional support for storing passwords in GNOME Keyring. pkg-config is needed to find GNOME Keyring headers and library. D-Bus and GLib are also required. If you want to build support for GNOME Keyring, then pass the '--with-gnome-keyring' option to `configure`. 20. Ctypesgen (OPTIONAL) Ctypesgen is Python wrapper generator for ctypes. It is used to generate a part of Subversion Ctypes Python bindings (CSVN). If you want to build CSVN, then pass the '--with-ctypesgen' option to `configure`. If ctypesgen.py is installed in a non-standard place, then use: --with-ctypesgen=/path/to/ctypesgen.py For more information on CSVN, see subversion/bindings/ctypes-python/README. 21. libmagic (OPTIONAL) Subversion's configure script attempts to find libmagic automatically. If it is installed in a non-standard location, then use: --with-libmagic=/path/to/libmagic/prefix The files include/magic.h and lib/libmagic.so.1.0 (or similar) are expected beneath this prefix directory. If they cannot be found Subversion will be compiled without support for libmagic. If libmagic is installed but support for it should not be compiled in, then use: --with-libmagic=no If configure should fail when libmagic is not present, but only the default locations should be searched, then use: --with-libmagic 22. LZ4 (OPTIONAL) Subversion uses LZ4 compression library version r129 or above. Configure will attempt to locate the system library by default using pkg-config and known paths. If it is installed in a non-standard location, then use: --with-lz4=/path/to/liblz4 If configure should use the version bundled with the sources, use: --with-lz4=internal 23. py3c (OPTIONAL) Subversion uses the Python 3 Compatibility Layer for C Extensions (py3c) library when building the Python language bindings. As py3c is a header-only library, it is needed only to build the bindings, not to use them. Configure will attempt to locate py3c by default using pkg-config and known paths. If it is installed in a non-standard location, then use: --with-py3c=/path/to/py3c/prefix The library can be downloaded from GitHub: https://github.com/encukou/py3c On Unix systems, you can also use the provided get-deps.sh script to download py3c and several other dependencies; see the top of section I.C for more about get-deps.sh. D. Documentation The primary documentation for Subversion is the free book "Version Control with Subversion", a.k.a. "The Subversion Book", obtainable from https://svnbook.red-bean.com/. Various additional documentation exists in the doc/ subdirectory of the Subversion source. See the file doc/README for more information. II. INSTALLATION ============ Subversion support three different build systems: - Autoconf/make, for Unix builds - Visual Studio vcproj, for Windows builds - CMake, for both Unix and Windows The first two have been in use since 2001. Sections A-E below describe the classic build system. The CMake build system was created in 2024 and is still under development. It will be included in Subversion 1.15 and is expected to be the default build system starting with Subversion 1.16. Section F below describes the CMake build system. A. Building from a Tarball ------------------------------ 1. Building from a Tarball Download the most recent distribution tarball from: https://subversion.apache.org/download/ Unpack it, and use the standard GNU procedure to compile: $ ./configure $ make # make install You can also run the full test suite by running 'make check'. Even in successful runs, some tests will report XFAIL; that is normal. Failed runs are indicated by FAIL or XPASS results, or a non-zero exit code from "make check". B. Building the Latest Source under Unix ------------------------------------- These instructions assume you have already installed Subversion and checked out a working copy of Subversion's own code -- either the latest /trunk code, or some branch or tag. You also need to have already installed whatever prerequisites that version of Subversion requires (if you haven't, the ./configure step should complain). You can discard the directory created by the tarball; you're about to build the latest, greatest Subversion client. This is the procedure Subversion developers use. First off, if you have any Subversion libraries lying around from previous 'make installs', clean them up first! # rm -f /usr/local/lib/libsvn* # rm -f /usr/local/lib/libapr* # rm -f /usr/local/lib/libserf* Start the process by running "autogen.sh": $ sh ./autogen.sh This script will make sure you have all the necessary components available to build Subversion. If any are missing, you will be told where to get them from. (See the 'Dependency Overview' in section I.) Note: if the command "autoconf" on your machine does not run autoconf 2.59 or later, but you do have a new enough autoconf available, then you can specify the correct one with the AUTOCONF variable. (The AUTOHEADER variable is similar.) This may be required on Debian GNU/Linux, where "autoconf" is actually a Perl script that attempts to guess which version is required -- because of the interaction between Subversion's and APR's configuration systems, the Perl script may get it wrong. So for example, you might need to do: $ AUTOCONF=autoconf2.59 sh ./autogen.sh Once you've prepared the working copy by running autogen.sh, just follow the usual configuration and build procedure: $ ./configure $ make # make install (Optionally, you might want to pass --enable-maintainer-mode to the ./configure script. This enables debugging symbols in your binaries (among other things) and most Subversion developers use it.) Since the resulting binary depends on shared libraries, the destination library directory must be identified in your operating system's library search path. That is in either /etc/ld.so.conf or $LD_LIBRARY_PATH for Linux systems and in /etc/rc.conf for FreeBSD, followed by a run of the 'ldconfig' program. Check your system documentation for details. By identifying the destination directory, Subversion will be able to dynamically load repository access plugins. If you try to do a checkout and see an error like: subversion/libsvn_ra/ra_loader.c:209: (apr_err=170000) svn: Unrecognized URL scheme 'https://svn.apache.org/repos/asf/subversion/trunk' It probably means that the dynamic loader/linker can't find all of the libsvn_* libraries. C. Building under Unix in Different Directories -------------------------------------------- It is possible to configure and build Subversion on Unix in a directory other than the working copy. For example $ svn co https://svn.apache.org/repos/asf/subversion/trunk svn $ cd svn $ # get SQLite amalgamation if required $ chmod +x autogen.sh $ ./autogen.sh $ mkdir ../obj $ cd ../obj $ ../svn/configure [...with options as appropriate...] $ make puts the Subversion working copy in the directory svn and builds it in a separate, parallel directory obj. Why would you want to do this? Well there are a number of reasons... * You may prefer to avoid "polluting" the working copy with files generated during the build. * You may want to put the build directory and the working copy on different physical disks to improve performance. * You may want to separate source and object code and only backup the source. * You may want to remote mount the working copy on multiple machines, and build for different machines from the same working copy. * You may want to build multiple configurations from the same working copy. The last reason above is possibly the most useful. For instance you can have separate debug and optimized builds each using the same working copy. Or you may want a client-only build and a client-server build. Using multiple build directories you can rebuild any or all configurations after an edit without the need to either clean and reconfigure, or identify and copy changes into another working copy. D. Installing from a Zip or Installer File under Windows ----------------------------------------------------- Of all the ways of getting a Subversion client, this is the easiest. Download a Zip or self-extracting installer via: https://subversion.apache.org/packages.html#windows For a Zip file extract the DLLs and EXEs to a directory of your choice. Included in the download are among other tools the SVN client, the SVNADMIN administration tool and the SVNLOOK reporting tool. You may want to add the bin directory in the Subversion folder to your PATH environment variable so as to not have to use the full path when running Subversion commands. To test the installation, open a DOS box (run either "cmd" or "command" from the Start menu's "Run..." menu option), change to the directory you installed the executables into, and run: C:\test>svn co https://svn.apache.org/repos/asf/subversion/trunk svn This will get the latest Subversion sources and put them into the "svn" subdirectory. If using a self-extracting .exe file, just run it instead of unzipping it, to install Subversion. E. Building the Latest Source under Windows ---------------------------------------- E.1 Prerequisites * Microsoft Visual Studio. Any recent (2005+) version containing the Visual C++ component will work (E.g. Professional, Express, Community Edition). Make sure you enable C++ support during setup. * Python 2.7 or higher, downloaded from https://www.python.org/ which is used to generate the project files. * Perl 5.8 or higher from https://www.perl.org/get.html * Awk is needed to compile Apache. Source code is available in tools\dev\awk, run the buildwin.bat program to compile. * Apache apr, apr-util, and optionally apr-iconv libraries, version 1.4 or later (1.2 for apr-iconv). If you are building from a Subversion checkout and have not downloaded Apache 2, then get these 3 libraries from https://www.apache.org/dist/apr/. * SQLite 3.24.0 or higher from https://www.sqlite.org/download.html (3.39.4 or higher recommended) * ZLib 1.2 or higher is required and can be obtained from http://www.zlib.net/ * Either a Subversion client binary from https://subversion.apache.org/packages.html to do the initial checkout of the Subversion source or the zip file source distribution. Additional Options * [Optional] Apache Httpd 2 source, downloaded from https://httpd.apache.org/download.cgi, these instructions assume version 2.0.58. This is only needed for building the Subversion server Apache modules. ### FIXME Apache 2.2 or greater required. * [Optional] Berkeley DB for backend support of the server components are available from http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index-082944.html (Version 4.4.20 or in specific cases some higher version recommended) For more information see Section I.C.9. * [Optional] Openssl can be obtained from https://www.openssl.org/source/ * [Optional] NASM can be obtained from http://www.nasm.us/ * [Optional] A modified version of GNU libintl, called svn-win32-libintl.zip, can be used for displaying localized messages. Available at: http://subversion.tigris.org/servlets/ProjectDocumentList?folderID=2627 * [Optional] GNU gettext for generating message catalog (.mo) files from message translations. You can get the latest binaries from http://gnuwin32.sourceforge.net/. You'll need the binaries (gettext-0.14.1-bin.zip) and dependencies (gettext-0.14.1-dep.zip). E.2 Notes The Apache Serf library supports secure connections with OpenSSL and on-the-wire compression with zlib. If you want to use the secure connections feature, you should pass the option "--with-openssl" to the gen-make.py script. See Section I.C.7 for more details. E.3 Preparation This section describes how to unpack the files to make a build tree. * Make a directory SVN and cd into it. * Either checkout Subversion: svn co https://svn.apache.org/repos/asf/subversion/trunk src-trunk or unpack the zip file distribution and rename the directory to src-trunk. * Install Visual Studio Environment. You either have to tell the installer to register environment variables or run VCVARS32.BAT before building anything. If you are using a newer Visual Studio, use the 'Visual Studio 20xx Command Prompt' on the Start menu. * Install Python and add it to your path * Install Perl (it should add itself to the path) ### Subversion doesn't need perl. Only some dependencies need it (OpenSSL and some apr scripts) * Copy AWK (awk95.exe) to awk.exe (e.g. SVN\awk\awk.exe) and add the directory containing it (e.g. SVN\awk) to the path. ### Subversion doesn't need awk. Only some dependencies need it (some apr scripts) * [Optional] Install NASM and add it to your path ### Subversion doesn't need NASM. Only some dependencies need it optionally (OpenSSL) * [Optional] If you checked out Subversion from the repository and want to build Subversion with http/https access support then install the Apache Serf sources into SVN\src-trunk\serf. * [Optional] If you want BDB backend support, extract the Berkeley DB files into SVN\src-trunk\db4-win32. It's a good idea to add SVN\src-trunk\db4-win32\bin to your PATH, so that Subversion can find the Berkeley DB DLLs. [NOTE: This binary package of Berkeley DB is provided for convenience only. Please don't address questions about Berkeley DB that aren't directly related to using Subversion to the project mailing list.] If you build Berkeley DB from the source, you will have to copy the file db-x.x.x\build_win32\db.h to SVN\src-trunk\db4-win32\include, and all the import libraries to SVN\src-trunk\db4-win32\lib. Again, the DLLs should be somewhere in your path. ### Just use --with-serf instead of the hardcoded path * [Optional] If you want to build the server modules, extract Apache source into SVN\httpd-2.x.x. * If you are building from a checkout of Subversion, and you are NOT building Apache, then you will need the APR libraries. Depending on how you got your version of APR, either: - Extract the APR, APR-util and APR-iconv source distributions into SVN\apr, SVN\apr-util, and SVN\apr-iconv respectively. Or: - Extract the apr, apr-util and apr-iconv directories from the srclib folder in the Apache httpd source into SVN\apr, SVN\apr-util, and SVN\apr-iconv respectively. ### Just use --with-apr, etc. instead of the hardcoded paths * Extract the ZLib sources into SVN\zlib if you are not using the zlib included in the dependencies zip file. ### Just use --with-zlib instead of the hardcoded path * [Optional] If you want secure connection (https) client support extract OpenSSL into SVN\openssl ### And pass the path to both serf and gen-make.py * [Optional] If you want localized message support, extract svn-win32-libintl.zip into SVN\svn-win32-libintl and extract gettext-x.x.x-bin.zip and gettext-x.x.x-dep.zip into SVN\gettext-x.x.x-bin. Add SVN\gettext-x.x.x-bin\bin to your path. * Download the SQLite amalgamation from https://www.sqlite.org/download.html and extract it into SVN\sqlite-amalgamation. See I.C.12 for alternatives to using the amalgamation package. E.4 Building the Binaries To build the binaries either follow these instructions. Start in the SVN directory you created. Set up the environment (commands should be one line even if wrapped here). C:>set VER=trunk C:>set DIR=trunk C:>set BUILD_ROOT=C:\SVN C:>set PYTHONDIR=C:\Python27 C:>set AWKDIR=C:\SVN\Awk C:>set ASMDIR=C:\SVN\asm C:>set SDKINC="C:\Program Files\Microsoft SDK\include" C:>set SDKLIB="C:\Program Files\Microsoft SDK\lib" C:>set GETTEXTBIN=C:\SVN\gettext-0.14.1-bin\bin C:>PATH=%PATH%;%BUILD_ROOT%\src-%DIR%\db4-win32;%ASMDIR%; %PYTHONDIR%;%AWKDIR%;%GETTEXTBIN% C:>set INCLUDE=%SDKINC%;%INCLUDE% C:>set LIB=%SDKLIB%;%LIB% OpenSSL < 1.1.0 C:>cd openssl C:>perl Configure VC-WIN32 [*] C:>call ms\do_masm C:>nmake -f ms\ntdll.mak C:>cd out32dll C:>call ..\ms\test C:>cd ..\.. *Note: Use "call ms\do_nasm" if you have nasm instead of MASM, or "call ms\do_ms" if you don't have an assembler. Also if you are using OpenSSL >= 1.0.0 masm is no longer supported. You will have to use do_nasm or do_ms in this case. OpenSSL >= 1.1.0 C:>cd openssl C:>perl Configure VC-WIN32 C:>nmake C:>nmake test C:>cd .. Apache 2 This step is only required for building the server dso modules. ### FIXME Apache 2.2 or greater required. Old build instructions for VC6. C:>set APACHEDIR=C:\Program Files\Apache Group\Apache2 C:>msdev httpd-2.0.58\apache.dsw /MAKE "BuildBin - Win32 Release" APR If you downloaded APR / APR-UTIL / APR_ICONV by source, you will have to build these libraries first. Building these libraries on Windows is straight forward and in most cases as simple as issuing these two commands: C:>nmake -f Makefile.win C:>nmake -f Makefile.win install Please refer to the build instructions provided by the library source for actual build instructions. ZLib If you downloaded the zlib source, you will have to build ZLib first. Building ZLib using Visual Studio should be quite simple. Just open the appropriate solution and build the project zlibstat using the IDE. Please refer to the build instructions provided by the library source for actual build instructions. Note that you'd make sure to define ZLIB_WINAPI in the ZLib config header and move the lib-file into the zlib root-directory. Please note that you MUST NOT build ZLib with the included assembler optimized code. It is known to be buggy, see for example the discussion https://svn.haxx.se/dev/archive-2013-10/0109.shtml. This means that you must not define ASMV or ASMINF. Note that the VS projects in contrib\visualstudio define these in the Debug configuration. Apache Serf ### Section about Apache Serf might be required/useful to add. ### scons is required too and Apache Serf needs to be configured prior to ### be able to build Subversion using: ### scons APR=[PATH_TO_APR] APU=[PATH_TO_APU] OPENSSL=[PATH_TO_OPENSSL] ### ZLIB=[PATH_TO_ZLIB] PREFIX=[PATH_TO_SERF_DEST] ### scons check ### scons install Subversion Things to note: * If you don't want to build mod_dav_svn, omit the --with-httpd option. The zip file source distribution contains apr, apr-util and apr-iconv in the default build location. If you have downloaded the apr files yourself you will have to tell the generator where to find the APR libraries; the options are --with-apr, --with-apr-util and --with-apr-iconv. * If you would like a debug build substitute Debug for Release in the msbuild command. * There have been rumors that Subversion on Win32 can be built using the latest cygwin, you probably don't want the zip file source distribution though. ymmv. * You will also have to distribute the C runtime dll with the binaries. Also, since Apache/APR do not provide .vcproj files, you will need to convert the Apache/APR .dsp files to .vcproj files with Visual Studio before building -- just open the Apache .dsw file and answer 'Yes To All' when the conversion dialog pops up, or you can open the individual .dsp files and convert them one at a time. The Apache/APR projects required by Subversion are: apr-util\libaprutil.dsp, apr\libapr.dsp, apr-iconv\libapriconv.dsp, apr-util\xml\expat\lib\xml.dsp, apr-iconv\ccs\libapriconv_ccs_modules.dsp, and apr-iconv\ces\libapriconv_ces_modules.dsp. * If the server dso modules are being built and tested Apache must not be running or the copy of the dso modules will fail. C:>cd src-%DIR% If Apache 2 has been built and the server modules are required then gen-make.py will already have been run. If the source is from the zip file, Apache 2 has not been built so gen-make.py must be run: C:>python gen-make.py --vsnet-version=20xx --with-berkeley-db=db4-win32 --with-openssl=..\openssl --with-zlib=..\zlib --with-libintl=..\svn-win32-libintl Then build subversion: C:>msbuild subversion_vcnet.sln /t:__MORE__ /p:Configuration=Release C:>cd .. The binaries have now been built. E.5 Packaging the binaries You now need to copy the binaries ready to make the release zip file. You also need to do this to run the tests as the new binaries need to be in your path. You can use the build/win32/make_dist.py script in the Subversion source directory to do that. [TBD: Describe how to do this. Note dependencies on zip, jar, doxygen.] E.6 Testing the Binaries [TBD: It's been a long, long while since it was necessary to move binaries around for testing. win-tests.py does that automagically. Fix this section accordingly, and probably reorder, putting the packaging at the end.] The build process creates the binary test programs but it does not copy the client tests into the release test area. C:>cd src-%DIR% C:>mkdir Release\subversion\tests\cmdline C:>xcopy /S /Y subversion\tests\cmdline Release\subversion\tests\cmdline If the server dso modules have been built then copy the dso files and dlls into the Apache modules directory. C:>copy Release\subversion\mod_dav_svn\mod_dav_svn.so "%APACHEDIR%"\modules C:>copy Release\subversion\mod_authz_svn\mod_authz_svn.so "%APACHEDIR%"\modules C:>copy svn-win32-%VER%\bin\intl.dll "%APACHEDIR%\bin" C:>copy svn-win32-%VER%\bin\iconv.dll "%APACHEDIR%\bin" C:>copy svn-win32-%VER%\bin\libdb42.dll "%APACHEDIR%\bin" C:>cd .. Put the svn-win32-trunk\bin directory at the start of your path so you run the newly built binaries and not another version you might have installed. Then run the client tests: C:>PATH=%BUILD_ROOT%\svn-win32-%VER%\bin;%PATH% C:>cd src-%DIR% C:>python win-tests.py -c -r -v If the server dso modules were built configure Apache to use the mod_dav_svn and mod_authz_svn modules by making sure these lines appear uncommented in httpd.conf: LoadModule dav_module modules/mod_dav.so LoadModule dav_fs_module modules/mod_dav_fs.so LoadModule dav_svn_module modules/mod_dav_svn.so LoadModule authz_svn_module modules/mod_authz_svn.so And further down the file add location directives to point to the test repositories. Change the paths to the SVN directory you created (paths should be on one line even if wrapped here): <Location /svn-test-work/repositories> DAV svn SVNParentPath C:/SVN/src-trunk/Release/subversion/tests/cmdline/ svn-test-work/repositories </Location> <Location /svn-test-work/local_tmp/repos> DAV svn SVNPath c:/SVN/src-trunk/Release/subversion/tests/cmdline/ svn-test-work/local_tmp/repos </Location> Then restart Apache and run the tests: C:>python win-tests.py -c -r -v -u http://localhost C:>cd .. F. Building using CMake -------------------- Get the sources, either a release tarball or by checking out the official repository. The CMake build system currently only exists in /trunk and it will be included in the 1.15 release. The process for building on Unix and Windows is the same. $ python gen-make.py -t cmake $ cmake -B out [build options] $ cmake --build out "out" in the commands above is the build directory used by CMake. Build options can be added, for example: $ cmake -B out -DCMAKE_INSTALL_PREFIX=/usr/local/subversion -DSVN_ENABLE_RA_SERF=ON Build options can be listed using: $ cmake -LH Windows tricks: - Modern versions of Microsoft Visual Studio provide support for CMake projects out-of-box, including intellisense, integrated options editor, test explorer, and more. In order to use it for Subversion, open the source directory with Visual Studio, and the configuration should start automatically. For editing the cache (options), do right-click to the CMakeLists.txt file and clicking `CMake Settings for Subversion` will open the editor. After the required settings are configured, hit `F7` in order to build. For more info, check the article bellow: https://learn.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio - There is a useful tool for bootstrapping the dependencies, vcpkg. It provides ports for the most of the Subversion's dependencies, which then could be installed via a single command. To start using it, download the registry from GitHub, bootstrap vcpkg, and install the dependencies: $ git clone https://github.com/microsoft/vcpkg $ cd vcpkg && .\bootstrap-vcpkg.bat -disableMetrics $ .\vcpkg install apr apr-util expat zlib sqlite3 [any other dependency] After this is done, vcpkg can be integrated into CMake by passing the vcpkg toolchain to CMAKE_TOOLCHAIN_FILE option. In order to do it with Visual Studio, open the CMake cache editor as explained in the previous step, and put the following into `CMake toolchain file` field, where VCPKG_ROOT is the path to vcpkg registry: <VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake III. BUILDING A SUBVERSION SERVER ============================ Subversion has two servers you can choose from: svnserve and Apache. svnserve is a small, lightweight server program that is automatically compiled when you build Subversion's source. Apache is a more heavyweight HTTP server, but tends to have more features. This section primarily focuses on how to build Apache and the accompanying mod_dav_svn server module for it. If you plan to use svnserve instead, jump right to section E for a quick explanation. A. Setting Up Apache Httpd ----------------------- 1. Obtaining and Installing Apache Httpd 2 Subversion tries to compile against the latest released version of Apache httpd 2.2+. The easiest thing for you to do is download a source tarball of the latest release and unpack that. If you have questions about the Apache httpd 2.2 build, please consult the httpd install documentation: https://httpd.apache.org/docs-2.2/install.html At the top of the httpd tree: $ ./buildconf $ ./configure --enable-dav --enable-so --enable-maintainer-mode The first arg says to build mod_dav. The second arg says to enable shared module support which is needed for a typical compile of mod_dav_svn (see below). The third arg says to include debugging information. If you built Subversion with --enable-maintainer-mode, then you should do the same for Apache; there can be problems if one was compiled with debugging and the other without. Note: if you have multiple db versions installed on your system, Apache might link to a different one than Subversion, causing failures when accessing the repository through Apache. To prevent this from happening, you have to tell Apache which db version to use and where to find db. Add --with-dbm=db4 and --with-berkeley-db=/usr/local/BerkeleyDB.4.2 to the configure line. Make sure this is the same db as the one Subversion uses. This note assumes you have installed Berkeley DB 4.2.52 at its default locations. For more info about the db requirement, see section I.C.9. You may also want to include other modules in your build. Add --enable-ssl to turn on SSL support, and --enable-deflate to turn on compression support, for example. Consult the Apache documentation for more details. All instructions below assume you configured Apache to install in its default location, /usr/local/apache2/; substitute appropriately if you chose some other location. Compile and install apache: $ make && make install B. Making and Installing the Subversion Apache Server Module --------------------------------------------------------- Go back into your subversion working copy and run ./autogen.sh if you need to. Then, assuming Apache httpd 2.2 is installed in the standard location, run: $ ./configure Note: do *not* configure subversion with "--disable-shared"! mod_dav_svn *must* be built as a shared library, and it will look for other libsvn_*.so libraries on your system. If you see a warning message that the build of mod_dav_svn is being skipped, this may be because you have Apache httpd 2.x installed in a non-standard location. You can use the "--with-apxs=" option to locate the apxs script: $ ./configure --with-apxs=/usr/local/apache2/bin/apxs Note: it *is* possible to build mod_dav_svn as a static library and link it directly into Apache. Possible, but painful. Stick with the shared library for now; if you can't, then ask. $ rm /usr/local/lib/libsvn* If you have old subversion libraries sitting on your system, libtool will link them instead of the `fresh' ones in your tree. Remove them before building subversion. $ make clean && make && make install After the make install, the Subversion shared libraries are in /usr/local/lib/. mod_dav_svn.so should be installed in /usr/local/libexec/ (or elsewhere, such as /usr/local/apache2/modules/, if you passed --with-apache-libexecdir to configure). Section II.E explains how to build the server on Windows. C. Configuring Apache Httpd for Subversion --------------------------------------- The following section is an abbreviated version of the information in the Subversion Book (https://svnbook.red-bean.com). Please read chapter 6 for more details. The following assumes you have already created a repository. For documentation on how to do that, see README. The following also assumes that you have modified /usr/local/apache2/conf/httpd.conf to reflect your setup. At a minimum you should look at the User, Group and ServerName directives. Full details on setting up apache can be found at: https://httpd.apache.org/docs-2.2/ First, your httpd.conf needs to load the mod_dav_svn module. If you pass --enable-mod-activation to Subversion's configure, 'make install' target should automatically add this line for you. In any case, if Apache HTTPD gives you an error like "Unknown DAV provider: svn", then you may want to verify that this line exists in your httpd.conf: LoadModule dav_svn_module modules/mod_dav_svn.so NOTE: if you built mod_dav as a dynamic module as well, make sure the above line appears after the one that loads mod_dav.so. Next, add this to the *bottom* of your httpd.conf: <Location /svn/repos> DAV svn SVNPath /absolute/path/to/repository </Location> This will give anyone unrestricted access to the repository. If you want limited access, read or write, you add these lines to the Location block: AuthType Basic AuthName "Subversion repository" AuthUserFile /my/svn/user/passwd/file And: a) For a read/write restricted repository: Require valid-user b) For a write restricted repository: <LimitExcept GET PROPFIND OPTIONS REPORT> Require valid-user </LimitExcept> c) For separate restricted read and write access: AuthGroupFile /my/svn/group/file <LimitExcept GET PROPFIND OPTIONS REPORT> Require group svn_committers </LimitExcept> <Limit GET PROPFIND OPTIONS REPORT> Require group svn_committers Require group svn_readers </Limit> ### FIXME Tutorials section refers to old 2.0 docs These are only a few simple examples. For a complete tutorial on Apache access control, please consider taking a look at the tutorials found under "Security" on the following page: https://httpd.apache.org/docs-2.0/misc/tutorials.html In order for 'svn cp' to work (which is actually implemented as a DAV COPY command), mod_dav needs to be able to determine the hostname of the server. A standard way of doing this is to use Apache's ServerName directive to set the server's hostname. Edit your /usr/local/apache2/conf/httpd.conf to include: ServerName svn.myserver.org If you are using virtual hosting through Apache's NameVirtualHost directive, you may need to use the ServerAlias directive to specify additional names that your server is known by. If you have configured mod_deflate to be in the server, you can enable compression support for your repository by adding the following line to your Location block: SetOutputFilter DEFLATE NOTE: If you are unfamiliar with an Apache directive, or not exactly sure about what it does, don't hesitate to look it up in the documentation: https://httpd.apache.org/docs-2.2/mod/directives.html. NOTE: Make sure that the user 'nobody' (or whatever UID the httpd process runs as) has permission to read and write the Berkeley DB files! This is a very common problem. D. Running and Testing ------------------- Fire up apache 2: $ /usr/local/apache2/bin/apachectl stop $ /usr/local/apache2/bin/apachectl start Check /usr/local/apache2/logs/error_log to make sure it started up okay. Try doing a network checkout from the repository: $ svn co http://localhost/svn/repos wc The most common reason this might fail is permission problems reading the repository db files. If the checkout fails, make sure that the httpd process has permission to read and write to the repository. You can see all of mod_dav_svn's complaints in the Apache error logfile, /usr/local/apache2/logs/error_log. To run the regression test suite for networked Subversion, see the instructions in subversion/tests/cmdline/README. For advice about tracing problems, see "Debugging the server" in https://subversion.apache.org/docs/community-guide/. E. Alternative: 'svnserve' and ra_svn ----------------------------------- An alternative network layer is libsvn_ra_svn (on the client side) and the 'svnserve' process on the server. This is a simple network layer that speaks a custom protocol over plain TCP (documented in libsvn_ra_svn/protocol): $ svnserve -d # becomes a background daemon $ svn checkout svn://localhost/usr/local/svn/repository You can use the "-r" option to svnserve to set a logical root for repositories, and the "-R" option to restrict connections to read-only access. ("Read-only" is a logical term here; svnserve still needs write access to the database in this mode, but will not allow commits or revprop changes.) 'svnserve' has built-in CRAM-MD5 authentication (so you can use non-system accounts), and can also be tunneled over SSH (so you can use existing system accounts). It's also capable of using Cyrus SASL if libsasl2 is detected at ./configure time. Please read chapter 6 in the Subversion Book (https://svnbook.red-bean.com) for details on these features. IV. PROGRAMMING LANGUAGE BINDINGS (PYTHON, PERL, RUBY, JAVA) ======================================================== For Python, Perl and Ruby bindings, see the file ./subversion/bindings/swig/INSTALL For Java bindings, see the file ./subversion/bindings/javahl/README
06-24
Quickstart Note The data files used in the quickstart guide are updated from time to time, which means that the adjusted close changes and with it the close (and the other components). That means that the actual output may be different to what was put in the documentation at the time of writing. Using the platform Let’s run through a series of examples (from almost an empty one to a fully fledged strategy) but not without before roughly explaining 2 basic concepts when working with backtrader Lines Data Feeds, Indicators and Strategies have lines. A line is a succession of points that when joined together form this line. When talking about the markets, a Data Feed has usually the following set of points per day: Open, High, Low, Close, Volume, OpenInterest The series of “Open”s along time is a Line. And therefore a Data Feed has usually 6 lines. If we also consider “DateTime” (which is the actual reference for a single point), we could count 7 lines. Index 0 Approach When accessing the values in a line, the current value is accessed with index: 0 And the “last” output value is accessed with -1. This in line with Python conventions for iterables (and a line can be iterated and is therefore an iterable) where index -1 is used to access the “last” item of the iterable/array. In our case is the last output value what’s getting accessed. As such and being index 0 right after -1, it is used to access the current moment in line. With that in mind and if we imagine a Strategy featuring a Simple Moving average created during initialization: self.sma = SimpleMovingAverage(.....) The easiest and simplest way to access the current value of this moving average: av = self.sma[0] There is no need to know how many bars/minutes/days/months have been processed, because “0” uniquely identifies the current instant. Following pythonic tradition, the “last” output value is accessed using -1: previous_value = self.sma[-1] Of course earlier output values can be accessed with -2, -3, … From 0 to 100: the samples Basic Setup Let’s get running. from __future__ import (absolute_import, division, print_function, unicode_literals) import backtrader as bt if __name__ == '__main__': cerebro = bt.Cerebro() print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) cerebro.run() print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 10000.00 Final Portfolio Value: 10000.00 In this example: backtrader was imported The Cerebro engine was instantiated The resulting cerebro instance was told to run (loop over data) And the resulting outcome was printed out Although it doesn’t seem much, let’s point out something explicitly shown: The Cerebro engine has created a broker instance in the background The instance already has some cash to start with This behind the scenes broker instantiation is a constant trait in the platform to simplify the life of the user. If no broker is set by the user, a default one is put in place. And 10K monetary units is a usual value with some brokers to begin with. Setting the Cash In the world of finance, for sure only “losers” start with 10k. Let’s change the cash and run the example again. from __future__ import (absolute_import, division, print_function, unicode_literals) import backtrader as bt if __name__ == '__main__': cerebro = bt.Cerebro() cerebro.broker.setcash(100000.0) print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) cerebro.run() print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 1000000.00 Final Portfolio Value: 1000000.00 Mission accomplished. Let’s move to tempestuous waters. Adding a Data Feed Having cash is fun, but the purpose behind all this is to let an automated strategy multiply the cash without moving a finger by operating on an asset which we see as a Data Feed Ergo … No Data Feed -> No Fun. Let’s add one to the ever growing example. from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values after this date todate=datetime.datetime(2000, 12, 31), reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 1000000.00 Final Portfolio Value: 1000000.00 The amount of boilerplate has grown slightly, because we added: Finding out where our example script is to be able to locate the sample Data Feed file Having datetime objects to filter on which data from the Data Feed we will be operating Aside from that, the Data Feed is created and added to cerebro. The output has not changed and it would be a miracle if it had. Note Yahoo Online sends the CSV data in date descending order, which is not the standard convention. The reversed=True prameter takes into account that the CSV data in the file has already been reversed and has the standard expected date ascending order. Our First Strategy The cash is in the broker and the Data Feed is there. It seems like risky business is just around the corner. Let’s put a Strategy into the equation and print the “Close” price of each day (bar). DataSeries (the underlying class in Data Feeds) objects have aliases to access the well known OHLC (Open High Low Close) daily values. This should ease up the creation of our printing logic. from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): def log(self, txt, dt=None): ''' Logging function for this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 100000.00 2000-01-03T00:00:00, Close, 27.85 2000-01-04T00:00:00, Close, 25.39 2000-01-05T00:00:00, Close, 24.05 ... ... ... 2000-12-26T00:00:00, Close, 29.17 2000-12-27T00:00:00, Close, 28.94 2000-12-28T00:00:00, Close, 29.29 2000-12-29T00:00:00, Close, 27.41 Final Portfolio Value: 100000.00 Someone said the stockmarket was risky business, but it doesn’t seem so. Let’s explain some of the magic: Upon init being called the strategy already has a list of datas that are present in the platform This is a standard Python list and datas can be accessed in the order they were inserted. The first data in the list self.datas[0] is the default data for trading operations and to keep all strategy elements synchronized (it’s the system clock) self.dataclose = self.datas[0].close keeps a reference to the close line. Only one level of indirection is later needed to access the close values. The strategy next method will be called on each bar of the system clock (self.datas[0]). This is true until other things come into play like indicators, which need some bars to start producing an output. More on that later. Adding some Logic to the Strategy Let’s try some crazy idea we had by looking at some charts If the price has been falling 3 sessions in a row … BUY BUY BUY!!! from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): def log(self, txt, dt=None): ''' Logging function fot this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) if self.dataclose[0] < self.dataclose[-1]: # current close less than previous close if self.dataclose[-1] < self.dataclose[-2]: # previous close less than the previous close # BUY, BUY, BUY!!! (with all possible default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) self.buy() if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 100000.00 2000-01-03, Close, 27.85 2000-01-04, Close, 25.39 2000-01-05, Close, 24.05 2000-01-05, BUY CREATE, 24.05 2000-01-06, Close, 22.63 2000-01-06, BUY CREATE, 22.63 2000-01-07, Close, 24.37 ... ... ... 2000-12-20, BUY CREATE, 26.88 2000-12-21, Close, 27.82 2000-12-22, Close, 30.06 2000-12-26, Close, 29.17 2000-12-27, Close, 28.94 2000-12-27, BUY CREATE, 28.94 2000-12-28, Close, 29.29 2000-12-29, Close, 27.41 Final Portfolio Value: 99725.08 Several “BUY” creation orders were issued, our porftolio value was decremented. A couple of important things are clearly missing. The order was created but it is unknown if it was executed, when and at what price. The next example will build upon that by listening to notifications of order status. The curious reader may ask how many shares are being bought, what asset is being bought and how are orders being executed. Where possible (and in this case it is) the platform fills in the gaps: self.datas[0] (the main data aka system clock) is the target asset if no other one is specified The stake is provided behind the scenes by a position sizer which uses a fixed stake, being the default “1”. It will be modified in a later example The order is executed “At Market”. The broker (shown in previous examples) executes this using the opening price of the next bar, because that’s the 1st tick after the current under examination bar. The order is executed so far without any commission (more on that later) Do not only buy … but SELL After knowing how to enter the market (long), an “exit concept” is needed and also understanding whether the strategy is in the market. Luckily a Strategy object offers access to a position attribute for the default data feed Methods buy and sell return the created (not yet executed) order Changes in orders’ status will be notified to the strategy via a notify method The “exit concept” will be an easy one: Exit after 5 bars (on the 6th bar) have elapsed for good or for worse Please notice that there is no “time” or “timeframe” implied: number of bars. The bars can represent 1 minute, 1 hour, 1 day, 1 week or any other time period. Although we know the data source is a daily one, the strategy makes no assumption about that. Additionally and to simplify: Do only allow a Buy order if not yet in the market Note The next method gets no “bar index” passed and therefore it seems obscure how to understand when 5 bars may have elapsed, but this has been modeled in pythonic way: call len on an object and it will tell you the length of its lines. Just write down (save in a variable) at which length in an operation took place and see if the current length is 5 bars away. from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): def log(self, txt, dt=None): ''' Logging function fot this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close # To keep track of pending orders self.order = None def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): self.log('BUY EXECUTED, %.2f' % order.executed.price) elif order.issell(): self.log('SELL EXECUTED, %.2f' % order.executed.price) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') # Write down: no pending order self.order = None def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one if self.order: return # Check if we are in the market if not self.position: # Not yet ... we MIGHT BUY if ... if self.dataclose[0] < self.dataclose[-1]: # current close less than previous close if self.dataclose[-1] < self.dataclose[-2]: # previous close less than the previous close # BUY, BUY, BUY!!! (with default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.buy() else: # Already in the market ... we might sell if len(self) >= (self.bar_executed + 5): # SELL, SELL, SELL!!! (with all possible default parameters) self.log('SELL CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.sell() if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 100000.00 2000-01-03T00:00:00, Close, 27.85 2000-01-04T00:00:00, Close, 25.39 2000-01-05T00:00:00, Close, 24.05 2000-01-05T00:00:00, BUY CREATE, 24.05 2000-01-06T00:00:00, BUY EXECUTED, 23.61 2000-01-06T00:00:00, Close, 22.63 2000-01-07T00:00:00, Close, 24.37 2000-01-10T00:00:00, Close, 27.29 2000-01-11T00:00:00, Close, 26.49 2000-01-12T00:00:00, Close, 24.90 2000-01-13T00:00:00, Close, 24.77 2000-01-13T00:00:00, SELL CREATE, 24.77 2000-01-14T00:00:00, SELL EXECUTED, 25.70 2000-01-14T00:00:00, Close, 25.18 ... ... ... 2000-12-15T00:00:00, SELL CREATE, 26.93 2000-12-18T00:00:00, SELL EXECUTED, 28.29 2000-12-18T00:00:00, Close, 30.18 2000-12-19T00:00:00, Close, 28.88 2000-12-20T00:00:00, Close, 26.88 2000-12-20T00:00:00, BUY CREATE, 26.88 2000-12-21T00:00:00, BUY EXECUTED, 26.23 2000-12-21T00:00:00, Close, 27.82 2000-12-22T00:00:00, Close, 30.06 2000-12-26T00:00:00, Close, 29.17 2000-12-27T00:00:00, Close, 28.94 2000-12-28T00:00:00, Close, 29.29 2000-12-29T00:00:00, Close, 27.41 2000-12-29T00:00:00, SELL CREATE, 27.41 Final Portfolio Value: 100018.53 Blistering Barnacles!!! The system made money … something must be wrong The broker says: Show me the money! And the money is called “commission”. Let’s add a reasonable 0.1% commision rate per operation (both for buying and selling … yes the broker is avid …) A single line will suffice for it: # 0.1% ... divide by 100 to remove the % cerebro.broker.setcommission(commission=0.001) Being experienced with the platform we want to see the profit or loss after a buy/sell cycle, with and without commission. from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): def log(self, txt, dt=None): ''' Logging function fot this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close # To keep track of pending orders and buy price/commission self.order = None self.buyprice = None self.buycomm = None def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): self.log( 'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.buyprice = order.executed.price self.buycomm = order.executed.comm else: # Sell self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') self.order = None def notify_trade(self, trade): if not trade.isclosed: return self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' % (trade.pnl, trade.pnlcomm)) def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one if self.order: return # Check if we are in the market if not self.position: # Not yet ... we MIGHT BUY if ... if self.dataclose[0] < self.dataclose[-1]: # current close less than previous close if self.dataclose[-1] < self.dataclose[-2]: # previous close less than the previous close # BUY, BUY, BUY!!! (with default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.buy() else: # Already in the market ... we might sell if len(self) >= (self.bar_executed + 5): # SELL, SELL, SELL!!! (with all possible default parameters) self.log('SELL CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.sell() if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) # Set the commission - 0.1% ... divide by 100 to remove the % cerebro.broker.setcommission(commission=0.001) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 100000.00 2000-01-03T00:00:00, Close, 27.85 2000-01-04T00:00:00, Close, 25.39 2000-01-05T00:00:00, Close, 24.05 2000-01-05T00:00:00, BUY CREATE, 24.05 2000-01-06T00:00:00, BUY EXECUTED, Price: 23.61, Cost: 23.61, Commission 0.02 2000-01-06T00:00:00, Close, 22.63 2000-01-07T00:00:00, Close, 24.37 2000-01-10T00:00:00, Close, 27.29 2000-01-11T00:00:00, Close, 26.49 2000-01-12T00:00:00, Close, 24.90 2000-01-13T00:00:00, Close, 24.77 2000-01-13T00:00:00, SELL CREATE, 24.77 2000-01-14T00:00:00, SELL EXECUTED, Price: 25.70, Cost: 25.70, Commission 0.03 2000-01-14T00:00:00, OPERATION PROFIT, GROSS 2.09, NET 2.04 2000-01-14T00:00:00, Close, 25.18 ... ... ... 2000-12-15T00:00:00, SELL CREATE, 26.93 2000-12-18T00:00:00, SELL EXECUTED, Price: 28.29, Cost: 28.29, Commission 0.03 2000-12-18T00:00:00, OPERATION PROFIT, GROSS -0.06, NET -0.12 2000-12-18T00:00:00, Close, 30.18 2000-12-19T00:00:00, Close, 28.88 2000-12-20T00:00:00, Close, 26.88 2000-12-20T00:00:00, BUY CREATE, 26.88 2000-12-21T00:00:00, BUY EXECUTED, Price: 26.23, Cost: 26.23, Commission 0.03 2000-12-21T00:00:00, Close, 27.82 2000-12-22T00:00:00, Close, 30.06 2000-12-26T00:00:00, Close, 29.17 2000-12-27T00:00:00, Close, 28.94 2000-12-28T00:00:00, Close, 29.29 2000-12-29T00:00:00, Close, 27.41 2000-12-29T00:00:00, SELL CREATE, 27.41 Final Portfolio Value: 100016.98 God Save the Queen!!! The system still made money. Before moving on, let’s notice something by filtering the “OPERATION PROFIT” lines: 2000-01-14T00:00:00, OPERATION PROFIT, GROSS 2.09, NET 2.04 2000-02-07T00:00:00, OPERATION PROFIT, GROSS 3.68, NET 3.63 2000-02-28T00:00:00, OPERATION PROFIT, GROSS 4.48, NET 4.42 2000-03-13T00:00:00, OPERATION PROFIT, GROSS 3.48, NET 3.41 2000-03-22T00:00:00, OPERATION PROFIT, GROSS -0.41, NET -0.49 2000-04-07T00:00:00, OPERATION PROFIT, GROSS 2.45, NET 2.37 2000-04-20T00:00:00, OPERATION PROFIT, GROSS -1.95, NET -2.02 2000-05-02T00:00:00, OPERATION PROFIT, GROSS 5.46, NET 5.39 2000-05-11T00:00:00, OPERATION PROFIT, GROSS -3.74, NET -3.81 2000-05-30T00:00:00, OPERATION PROFIT, GROSS -1.46, NET -1.53 2000-07-05T00:00:00, OPERATION PROFIT, GROSS -1.62, NET -1.69 2000-07-14T00:00:00, OPERATION PROFIT, GROSS 2.08, NET 2.01 2000-07-28T00:00:00, OPERATION PROFIT, GROSS 0.14, NET 0.07 2000-08-08T00:00:00, OPERATION PROFIT, GROSS 4.36, NET 4.29 2000-08-21T00:00:00, OPERATION PROFIT, GROSS 1.03, NET 0.95 2000-09-15T00:00:00, OPERATION PROFIT, GROSS -4.26, NET -4.34 2000-09-27T00:00:00, OPERATION PROFIT, GROSS 1.29, NET 1.22 2000-10-13T00:00:00, OPERATION PROFIT, GROSS -2.98, NET -3.04 2000-10-26T00:00:00, OPERATION PROFIT, GROSS 3.01, NET 2.95 2000-11-06T00:00:00, OPERATION PROFIT, GROSS -3.59, NET -3.65 2000-11-16T00:00:00, OPERATION PROFIT, GROSS 1.28, NET 1.23 2000-12-01T00:00:00, OPERATION PROFIT, GROSS 2.59, NET 2.54 2000-12-18T00:00:00, OPERATION PROFIT, GROSS -0.06, NET -0.12 Adding up the “NET” profits the final figure is: 15.83 But the system said the following at the end: 2000-12-29T00:00:00, SELL CREATE, 27.41 Final Portfolio Value: 100016.98 And obviously 15.83 is not 16.98. There is no error whatsoever. The “NET” profit of 15.83 is already cash in the bag. Unfortunately (or fortunately to better understand the platform) there is an open position on the last day of the Data Feed. Even if a SELL operation has been sent … IT HAS NOT YET BEEN EXECUTED. The “Final Portfolio Value” calculated by the broker takes into account the “Close” price on 2000-12-29. The actual execution price would have been set on the next trading day which happened to be 2001-01-02. Extending the Data Feed” to take into account this day the output is: 2001-01-02T00:00:00, SELL EXECUTED, Price: 27.87, Cost: 27.87, Commission 0.03 2001-01-02T00:00:00, OPERATION PROFIT, GROSS 1.64, NET 1.59 2001-01-02T00:00:00, Close, 24.87 2001-01-02T00:00:00, BUY CREATE, 24.87 Final Portfolio Value: 100017.41 Now adding the previous NET profit to the completed operation’s net profit: 15.83 + 1.59 = 17.42 Which (discarding rounding errors in the “print” statements) is the extra Portfolio above the initial 100000 monetary units the strategy started with. Customizing the Strategy: Parameters It would a bit unpractical to hardcode some of the values in the strategy and have no chance to change them easily. Parameters come in handy to help. Definition of parameters is easy and looks like: params = (('myparam', 27), ('exitbars', 5),) Being this a standard Python tuple with some tuples inside it, the following may look more appealling to some: params = ( ('myparam', 27), ('exitbars', 5), ) With either formatting parametrization of the strategy is allowed when adding the strategy to the Cerebro engine: # Add a strategy cerebro.addstrategy(TestStrategy, myparam=20, exitbars=7) Note The setsizing method below is deprecated. This content is kept here for anyone looking at old samples of the sources. The sources have been update to use: cerebro.addsizer(bt.sizers.FixedSize, stake=10)`` Please read the section about sizers Using the parameters in the strategy is easy, as they are stored in a “params” attribute. If we for example want to set the stake fix, we can pass the stake parameter to the position sizer like this durint init: # Set the sizer stake from the params self.sizer.setsizing(self.params.stake) We could have also called buy and sell with a stake parameter and self.params.stake as the value. The logic to exit gets modified: # Already in the market ... we might sell if len(self) >= (self.bar_executed + self.params.exitbars): With all this in mind the example evolves to look like: from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): params = ( ('exitbars', 5), ) def log(self, txt, dt=None): ''' Logging function fot this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close # To keep track of pending orders and buy price/commission self.order = None self.buyprice = None self.buycomm = None def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): self.log( 'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.buyprice = order.executed.price self.buycomm = order.executed.comm else: # Sell self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') self.order = None def notify_trade(self, trade): if not trade.isclosed: return self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' % (trade.pnl, trade.pnlcomm)) def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one if self.order: return # Check if we are in the market if not self.position: # Not yet ... we MIGHT BUY if ... if self.dataclose[0] < self.dataclose[-1]: # current close less than previous close if self.dataclose[-1] < self.dataclose[-2]: # previous close less than the previous close # BUY, BUY, BUY!!! (with default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.buy() else: # Already in the market ... we might sell if len(self) >= (self.bar_executed + self.params.exitbars): # SELL, SELL, SELL!!! (with all possible default parameters) self.log('SELL CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.sell() if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(100000.0) # Add a FixedSize sizer according to the stake cerebro.addsizer(bt.sizers.FixedSize, stake=10) # Set the commission - 0.1% ... divide by 100 to remove the % cerebro.broker.setcommission(commission=0.001) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) After the execution the output is: Starting Portfolio Value: 100000.00 2000-01-03T00:00:00, Close, 27.85 2000-01-04T00:00:00, Close, 25.39 2000-01-05T00:00:00, Close, 24.05 2000-01-05T00:00:00, BUY CREATE, 24.05 2000-01-06T00:00:00, BUY EXECUTED, Size 10, Price: 23.61, Cost: 236.10, Commission 0.24 2000-01-06T00:00:00, Close, 22.63 ... ... ... 2000-12-20T00:00:00, BUY CREATE, 26.88 2000-12-21T00:00:00, BUY EXECUTED, Size 10, Price: 26.23, Cost: 262.30, Commission 0.26 2000-12-21T00:00:00, Close, 27.82 2000-12-22T00:00:00, Close, 30.06 2000-12-26T00:00:00, Close, 29.17 2000-12-27T00:00:00, Close, 28.94 2000-12-28T00:00:00, Close, 29.29 2000-12-29T00:00:00, Close, 27.41 2000-12-29T00:00:00, SELL CREATE, 27.41 Final Portfolio Value: 100169.80 In order to see the difference, the print outputs have also been extended to show the execution size. Having multiplied the stake by 10, the obvious has happened: the profit and loss has been multiplied by 10. Instead of 16.98, the surplus is now 169.80 Adding an indicator Having heard of indicators, the next thing anyone would add to the strategy is one of them. For sure they must be much better than a simple “3 lower closes” strategy. Inspired in one of the examples from PyAlgoTrade a strategy using a Simple Moving Average. Buy “AtMarket” if the close is greater than the Average If in the market, sell if the close is smaller than the Average Only 1 active operation is allowed in the market Most of the existing code can be kept in place. Let’s add the average during init and keep a reference to it: self.sma = bt.indicators.MovingAverageSimple(self.datas[0], period=self.params.maperiod) And of course the logic to enter and exit the market will rely on the Average values. Look in the code for the logic. Note The starting cash will be 1000 monetary units to be in line with the PyAlgoTrade example and no commission will be applied from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): params = ( ('maperiod', 15), ) def log(self, txt, dt=None): ''' Logging function fot this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close # To keep track of pending orders and buy price/commission self.order = None self.buyprice = None self.buycomm = None # Add a MovingAverageSimple indicator self.sma = bt.indicators.SimpleMovingAverage( self.datas[0], period=self.params.maperiod) def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): self.log( 'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.buyprice = order.executed.price self.buycomm = order.executed.comm else: # Sell self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') self.order = None def notify_trade(self, trade): if not trade.isclosed: return self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' % (trade.pnl, trade.pnlcomm)) def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one if self.order: return # Check if we are in the market if not self.position: # Not yet ... we MIGHT BUY if ... if self.dataclose[0] > self.sma[0]: # BUY, BUY, BUY!!! (with all possible default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.buy() else: if self.dataclose[0] < self.sma[0]: # SELL, SELL, SELL!!! (with all possible default parameters) self.log('SELL CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.sell() if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(1000.0) # Add a FixedSize sizer according to the stake cerebro.addsizer(bt.sizers.FixedSize, stake=10) # Set the commission cerebro.broker.setcommission(commission=0.0) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) Now, before skipping to the next section LOOK CAREFULLY to the first date which is shown in the log: It’ no longer 2000-01-03, the first trading day in the year 2K. It’s 2000-01-24 … Who has stolen my cheese? The missing days are not missing. The platform has adapted to the new circumstances: An indicator (SimpleMovingAverage) has been added to the Strategy. This indicator needs X bars to produce an output: in the example: 15 2000-01-24 is the day in which the 15th bar occurs The backtrader platform assumes that the Strategy has the indicator in place for a good reason, to use it in the decision making process. And it makes no sense to try to make decisions if the indicator is not yet ready and producing values. next will be 1st called when all indicators have already reached the minimum needed period to produce a value In the example there is a single indicator, but the strategy could have any number of them. After the execution the output is: Starting Portfolio Value: 1000.00 2000-01-24T00:00:00, Close, 25.55 2000-01-25T00:00:00, Close, 26.61 2000-01-25T00:00:00, BUY CREATE, 26.61 2000-01-26T00:00:00, BUY EXECUTED, Size 10, Price: 26.76, Cost: 267.60, Commission 0.00 2000-01-26T00:00:00, Close, 25.96 2000-01-27T00:00:00, Close, 24.43 2000-01-27T00:00:00, SELL CREATE, 24.43 2000-01-28T00:00:00, SELL EXECUTED, Size 10, Price: 24.28, Cost: 242.80, Commission 0.00 2000-01-28T00:00:00, OPERATION PROFIT, GROSS -24.80, NET -24.80 2000-01-28T00:00:00, Close, 22.34 2000-01-31T00:00:00, Close, 23.55 2000-02-01T00:00:00, Close, 25.46 2000-02-02T00:00:00, Close, 25.61 2000-02-02T00:00:00, BUY CREATE, 25.61 2000-02-03T00:00:00, BUY EXECUTED, Size 10, Price: 26.11, Cost: 261.10, Commission 0.00 ... ... ... 2000-12-20T00:00:00, SELL CREATE, 26.88 2000-12-21T00:00:00, SELL EXECUTED, Size 10, Price: 26.23, Cost: 262.30, Commission 0.00 2000-12-21T00:00:00, OPERATION PROFIT, GROSS -20.60, NET -20.60 2000-12-21T00:00:00, Close, 27.82 2000-12-21T00:00:00, BUY CREATE, 27.82 2000-12-22T00:00:00, BUY EXECUTED, Size 10, Price: 28.65, Cost: 286.50, Commission 0.00 2000-12-22T00:00:00, Close, 30.06 2000-12-26T00:00:00, Close, 29.17 2000-12-27T00:00:00, Close, 28.94 2000-12-28T00:00:00, Close, 29.29 2000-12-29T00:00:00, Close, 27.41 2000-12-29T00:00:00, SELL CREATE, 27.41 Final Portfolio Value: 973.90 In the name of the King!!! A winning system turned into a losing one … and that with no commission. It may well be that simply adding an indicator is not the universal panacea. Note The same logic and data with PyAlgoTrade yields a slightly different result (slightly off). Looking at the entire printout reveals that some operations are not exactly the same. Being the culprit again the usual suspect: rounding. PyAlgoTrade does not round the datafeed values when applying the divided “adjusted close” to the data feed values. The Yahoo Data Feed provided by backtrader rounds the values down to 2 decimals after applying the adjusted close. Upon printing the values everything seems the same, but it’s obvious that sometimes that 5th place decimal plays a role. Rounding down to 2 decimals seems more realistic, because Market Exchanges do only allow a number of decimals per asset (being that 2 decimals usually for stocks) Note The Yahoo Data Feed (starting with version 1.8.11.99 allows to specify if rounding has to happen and how many decimals) Visual Inspection: Plotting A printout or log of the actual whereabouts of the system at each bar-instant is good but humans tend to be visual and therefore it seems right to offer a view of the same whereabouts as chart. Note To plot you need to have matplotlib installed Once again defaults for plotting are there to assist the platform user. Plotting is incredibly a 1 line operation: cerebro.plot() Being the location for sure after cerebro.run() has been called. In order to display the automatic plotting capabilities and a couple of easy customizations, the following will be done: A 2nd MovingAverage (Exponential) will be added. The defaults will plot it (just like the 1st) with the data. A 3rd MovingAverage (Weighted) will be added. Customized to plot in an own plot (even if not sensible) A Stochastic (Slow) will be added. No change to the defaults. A MACD will be added. No change to the defaults. A RSI will be added. No change to the defaults. A MovingAverage (Simple) will be applied to the RSI. No change to the defaults (it will be plotted with the RSI) An AverageTrueRange will be added. Changed defaults to avoid it being plotted. The entire set of additions to the init method of the Strategy: # Indicators for the plotting show bt.indicators.ExponentialMovingAverage(self.datas[0], period=25) bt.indicators.WeightedMovingAverage(self.datas[0], period=25).subplot = True bt.indicators.StochasticSlow(self.datas[0]) bt.indicators.MACDHisto(self.datas[0]) rsi = bt.indicators.RSI(self.datas[0]) bt.indicators.SmoothedMovingAverage(rsi, period=10) bt.indicators.ATR(self.datas[0]).plot = False Note Even if indicators are not explicitly added to a member variable of the strategy (like self.sma = MovingAverageSimple…), they will autoregister with the strategy and will influence the minimum period for next and will be part of the plotting. In the example only RSI is added to a temporary variable rsi with the only intention to create a MovingAverageSmoothed on it. The example now: from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): params = ( ('maperiod', 15), ) def log(self, txt, dt=None): ''' Logging function fot this strategy''' dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close # To keep track of pending orders and buy price/commission self.order = None self.buyprice = None self.buycomm = None # Add a MovingAverageSimple indicator self.sma = bt.indicators.SimpleMovingAverage( self.datas[0], period=self.params.maperiod) # Indicators for the plotting show bt.indicators.ExponentialMovingAverage(self.datas[0], period=25) bt.indicators.WeightedMovingAverage(self.datas[0], period=25, subplot=True) bt.indicators.StochasticSlow(self.datas[0]) bt.indicators.MACDHisto(self.datas[0]) rsi = bt.indicators.RSI(self.datas[0]) bt.indicators.SmoothedMovingAverage(rsi, period=10) bt.indicators.ATR(self.datas[0], plot=False) def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): self.log( 'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.buyprice = order.executed.price self.buycomm = order.executed.comm else: # Sell self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') # Write down: no pending order self.order = None def notify_trade(self, trade): if not trade.isclosed: return self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' % (trade.pnl, trade.pnlcomm)) def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one if self.order: return # Check if we are in the market if not self.position: # Not yet ... we MIGHT BUY if ... if self.dataclose[0] > self.sma[0]: # BUY, BUY, BUY!!! (with all possible default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.buy() else: if self.dataclose[0] < self.sma[0]: # SELL, SELL, SELL!!! (with all possible default parameters) self.log('SELL CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.sell() if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy cerebro.addstrategy(TestStrategy) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(1000.0) # Add a FixedSize sizer according to the stake cerebro.addsizer(bt.sizers.FixedSize, stake=10) # Set the commission cerebro.broker.setcommission(commission=0.0) # Print out the starting conditions print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Run over everything cerebro.run() # Print out the final result print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue()) # Plot the result cerebro.plot() After the execution the output is: Starting Portfolio Value: 1000.00 2000-02-18T00:00:00, Close, 27.61 2000-02-22T00:00:00, Close, 27.97 2000-02-22T00:00:00, BUY CREATE, 27.97 2000-02-23T00:00:00, BUY EXECUTED, Size 10, Price: 28.38, Cost: 283.80, Commission 0.00 2000-02-23T00:00:00, Close, 29.73 ... ... ... 2000-12-21T00:00:00, BUY CREATE, 27.82 2000-12-22T00:00:00, BUY EXECUTED, Size 10, Price: 28.65, Cost: 286.50, Commission 0.00 2000-12-22T00:00:00, Close, 30.06 2000-12-26T00:00:00, Close, 29.17 2000-12-27T00:00:00, Close, 28.94 2000-12-28T00:00:00, Close, 29.29 2000-12-29T00:00:00, Close, 27.41 2000-12-29T00:00:00, SELL CREATE, 27.41 Final Portfolio Value: 981.00 The final result has changed even if the logic hasn’t. This is true but the logic has not been applied to the same number of bars. Note As explained before, the platform will first call next when all indicators are ready to produce a value. In this plotting example (very clear in the chart) the MACD is the last indicator to be fully ready (all 3 lines producing an output). The 1st BUY order is no longer scheduled during Jan 2000 but close to the end of Feb 2000. The chart: image Let’s Optimize Many trading books say each market and each traded stock (or commodity or ..) have different rythms. That there is no such thing as a one size fits all. Before the plotting sample, when the strategy started using an indicator the period default value was 15 bars. It’s a strategy parameter and this can be used in an optimization to change the value of the parameter and see which one better fits the market. Note There is plenty of literature about Optimization and associated pros and cons. But the advice will always point in the same direction: do not overoptimize. If a trading idea is not sound, optimizing may end producing a positive result which is only valid for the backtested dataset. The sample is modified to optimize the period of the Simple Moving Average. For the sake of clarity any output with regards to Buy/Sell orders has been removed The example now: from __future__ import (absolute_import, division, print_function, unicode_literals) import datetime # For datetime objects import os.path # To manage paths import sys # To find out the script name (in argv[0]) # Import the backtrader platform import backtrader as bt # Create a Stratey class TestStrategy(bt.Strategy): params = ( ('maperiod', 15), ('printlog', False), ) def log(self, txt, dt=None, doprint=False): ''' Logging function fot this strategy''' if self.params.printlog or doprint: dt = dt or self.datas[0].datetime.date(0) print('%s, %s' % (dt.isoformat(), txt)) def __init__(self): # Keep a reference to the "close" line in the data[0] dataseries self.dataclose = self.datas[0].close # To keep track of pending orders and buy price/commission self.order = None self.buyprice = None self.buycomm = None # Add a MovingAverageSimple indicator self.sma = bt.indicators.SimpleMovingAverage( self.datas[0], period=self.params.maperiod) def notify_order(self, order): if order.status in [order.Submitted, order.Accepted]: # Buy/Sell order submitted/accepted to/by broker - Nothing to do return # Check if an order has been completed # Attention: broker could reject order if not enough cash if order.status in [order.Completed]: if order.isbuy(): self.log( 'BUY EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.buyprice = order.executed.price self.buycomm = order.executed.comm else: # Sell self.log('SELL EXECUTED, Price: %.2f, Cost: %.2f, Comm %.2f' % (order.executed.price, order.executed.value, order.executed.comm)) self.bar_executed = len(self) elif order.status in [order.Canceled, order.Margin, order.Rejected]: self.log('Order Canceled/Margin/Rejected') # Write down: no pending order self.order = None def notify_trade(self, trade): if not trade.isclosed: return self.log('OPERATION PROFIT, GROSS %.2f, NET %.2f' % (trade.pnl, trade.pnlcomm)) def next(self): # Simply log the closing price of the series from the reference self.log('Close, %.2f' % self.dataclose[0]) # Check if an order is pending ... if yes, we cannot send a 2nd one if self.order: return # Check if we are in the market if not self.position: # Not yet ... we MIGHT BUY if ... if self.dataclose[0] > self.sma[0]: # BUY, BUY, BUY!!! (with all possible default parameters) self.log('BUY CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.buy() else: if self.dataclose[0] < self.sma[0]: # SELL, SELL, SELL!!! (with all possible default parameters) self.log('SELL CREATE, %.2f' % self.dataclose[0]) # Keep track of the created order to avoid a 2nd order self.order = self.sell() def stop(self): self.log('(MA Period %2d) Ending Value %.2f' % (self.params.maperiod, self.broker.getvalue()), doprint=True) if __name__ == '__main__': # Create a cerebro entity cerebro = bt.Cerebro() # Add a strategy strats = cerebro.optstrategy( TestStrategy, maperiod=range(10, 31)) # Datas are in a subfolder of the samples. Need to find where the script is # because it could have been called from anywhere modpath = os.path.dirname(os.path.abspath(sys.argv[0])) datapath = os.path.join(modpath, '../../datas/orcl-1995-2014.txt') # Create a Data Feed data = bt.feeds.YahooFinanceCSVData( dataname=datapath, # Do not pass values before this date fromdate=datetime.datetime(2000, 1, 1), # Do not pass values before this date todate=datetime.datetime(2000, 12, 31), # Do not pass values after this date reverse=False) # Add the Data Feed to Cerebro cerebro.adddata(data) # Set our desired cash start cerebro.broker.setcash(1000.0) # Add a FixedSize sizer according to the stake cerebro.addsizer(bt.sizers.FixedSize, stake=10) # Set the commission cerebro.broker.setcommission(commission=0.0) # Run over everything cerebro.run(maxcpus=1) Instead of calling addstrategy to add a stratey class to Cerebro, the call is made to optstrategy. And instead of passing a value a range of values is passed. One of the “Strategy” hooks is added, the stop method, which will be called when the data has been exhausted and backtesting is over. It’s used to print the final net value of the portfolio in the broker (it was done in Cerebro previously) The system will execute the strategy for each value of the range. The following will be output: 2000-12-29, (MA Period 10) Ending Value 880.30 2000-12-29, (MA Period 11) Ending Value 880.00 2000-12-29, (MA Period 12) Ending Value 830.30 2000-12-29, (MA Period 13) Ending Value 893.90 2000-12-29, (MA Period 14) Ending Value 896.90 2000-12-29, (MA Period 15) Ending Value 973.90 2000-12-29, (MA Period 16) Ending Value 959.40 2000-12-29, (MA Period 17) Ending Value 949.80 2000-12-29, (MA Period 18) Ending Value 1011.90 2000-12-29, (MA Period 19) Ending Value 1041.90 2000-12-29, (MA Period 20) Ending Value 1078.00 2000-12-29, (MA Period 21) Ending Value 1058.80 2000-12-29, (MA Period 22) Ending Value 1061.50 2000-12-29, (MA Period 23) Ending Value 1023.00 2000-12-29, (MA Period 24) Ending Value 1020.10 2000-12-29, (MA Period 25) Ending Value 1013.30 2000-12-29, (MA Period 26) Ending Value 998.30 2000-12-29, (MA Period 27) Ending Value 982.20 2000-12-29, (MA Period 28) Ending Value 975.70 2000-12-29, (MA Period 29) Ending Value 983.30 2000-12-29, (MA Period 30) Ending Value 979.80 Results: For periods below 18 the strategy (commissionless) loses money. For periods between 18 and 26 (both included) the strategy makes money. Above 26 money is lost again. And the winning period for this strategy and the given data set is: 20 bars, which wins 78.00 units over 1000 $/€ (a 7.8%) Note The extra indicators from the plotting example have been removed and the start of operations is only influenced by the Simple Moving Average which is being optimized. Hence the slightly different results for period 15 Conclusion The incremental samples have shown how to go from a barebones script to a fully working trading system which even plots the results and can be optimized. A lot more can be done to try to improve the chances of winning: Self defined Indicators Creating an indicator is easy (and even plotting them is easy) Sizers Money Management is for many the key to success Order Types (limit, stop, stoplimit) Some others To ensure all the above items can be fully utilized the documentation provides an insight into them (and other topics) Look in the table of contents and keep on reading … and developing. Best of luck
最新发布
07-08
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值