Swift/T Guide

name

wget http://www.swift-lang.org/Swift-T/downloads/exm-0.7.0.tar.gz
tar xfz exm-trunk.tar.gz
cd exm-trunk
./exm-setup.zsh
PATH=${PATH}:/path/to/exm-install/turbine/bin
PATH=${PATH}:/path/to/exm-install/stc/bin
stc program.swift
turbine -n 4 program.tcl
swift-t -n 4 program.swift
import io;

printf("Hello world");
// This is a comment
/* This is a
comment */
/** Also a
comment */
# This will work if source file is not preprocessed
import io;
import mypackage.mymodule;
#include <mypackage/mymodule.swift>
<type> <variable name>;
<type> <variable name> = <expression>;
<variable name> = <expression>;
int x;
int x = 1;
x = 1;
int x;
{
  int x;
}
{
  int x;
}
{
  int x;
}
int z1,z2;
int y;
int x = f(y);
y = g(2);
z1 = h(x,y,1);
z2 = h(x,y,2);
int output = r(z1,z2);
[(<output list>)] function_name [(<input list>)]
{
  statement;
  statement;
  ...
}
x1, x2 = f(i1, i2);
// or equivalently:
(x1, x2) = f(i1, i2);
string s =
"""
line data 1
line data 2
""";
file x = input_file("/home/user/in.txt");
file y <"/home/user/out.txt">; // Declare a mapped file
y = x; // Do the copy
// Initialize an array of files from a range of files on disk with glob
file f[] = glob("directory/*.txt");

// Read the contents of a file with read
string filename = "in.txt";
string contents = read(input_file(filename));
trace("Contents of " + filename + ":\n" + contents);

// Write directly to a file with write
file tmp = write("first line\nsecond line");

// Find the name of a file with filename
trace("Temporary filename is: " + filename(tmp));
file f<"f.txt"> = g();
file f<"f.txt">=g();
int A[];
string dict[string];
int A[string];
int B[];
B = function_returning_array();
A["zero"] = B[0];
A["one"] = B[1];
(int B[]) f(int j)
{
  int A[];
  A = subroutine_function(1);

  // Error: A has already been assigned in toto:
  A[3] = 4;

  // OK: assigning to output variable
  B = subroutine_function(2);
}
int start = 0;
int stop = 10;
int step = 2;
// Array of length 10:
int A[] = [start:stop-1];
// Array of length 5, containing only even numbers:
int B[] = [start:stop-1:step];
int C[] = [4,5,6];
// An array of arrays of files with string keys
file A[string][string];
A["foo"]["bar"] = input_file("test.txt");
A["foo"]["qux"] = input_file("test2.txt");
type person
{
  string name;
  int age;
  int events[];
}
person p;
p.name = "Abe";
p.age = 90;
person people[], p1, p2;

p1.name = "Thelma";
p1.age = 31;

p2.name = "Louise";
p2.age = 29;

people[0] = p1;
people[1] = p2;
people[2].name = "Abe";  // Not supported!
people[2].age = 90;      // Not supported!
person p3 = person("Nguyen", 42);
typedef newint int;

// We can freely convert between int and newint
newint x = 1;
int y = x;
newint z = y;
typedef sorted_file file;
app (sorted_file out) sort (file i) {
  "/usr/bin/sort" "-o" out i
}

// uniq utility requires sorted input
app (file o) unique (sorted_file i) {
  "/usr/bin/uniq" i @stdout=o
}

file unsorted = input_file("input.txt");
sorted_file sorted <"sorted.txt"> = sort(unsorted);
file u <"unique.txt"> = unique(sorted);

// Can convert from sorted_file to file
file result2 = sort(unsorted);

// This would cause a type error
// sorted_file not_sorted = unsorted;
string HELLO = "Hello World";
float PI_APPROX = 3.142;
int ONE = 1;

trace(HELLO, PI_APPROX, ONE);
if (<condition>)
{
  statement;
  ...
}
else
{
  statement;
  ...
}
int a = 20;
switch (a)
{
  case 1:
    int c;
    c = a + a;
    b = a + 1;
  case 20:
    b = 1;
  case 2000:
    b = 2;
  default:
    b = 2102 + 2420;
}
printf("b: %i\n", b);
string A[];
foreach value, index in A
{
  printf("A[%i] = %s\n", index, value);
}
foreach i in [start:stop:step] {
    ...
}
int N = 100;
int count = 0;
for (int i = 0; i < N; i = i+1, count = count+c)
{
  int c;
  if (condition_function(i))
  {
    c = 1;
  }
  else
  {
    c = 0;
  }
}
for ( <initializer> ; <condition> ; <updates> )
{
  statement;
  ...
}
x = f();
y = g();
wait (x) {
  trace("x is closed!");
}
wait(x, y) {
  trace("x and y are closed!");
}
sleep(1) =>
  x = f() =>
  int y = g() =>
  trace("DONE!");
{
  int b;
  b = 1;
}
{
  int b;
  b = 2;
}
turbine -l -n 3 program.tcl -v -a=file1.txt file2.txt --exec="prog thing1 thing2" --help file4.txt
() my_output (string s) "turbine" "0.0" [
  "puts <<s>>"
];

my_output("HELLO");
namespace eval myextension {
  proc double { x } {
    return [ expr $x * 2 ]
  }
}
@pure
(int o) double (int i) "myextension" "0.0.1" [
  "set <<o>> [ myextension::double <<i>> ]"
];
@dispatch=WORKER
(int o) process (int i) "pkg" "0.0.1" [
  "set <<o>> [ pkg::process <<i>> ]"
];
stc -r <package directory> ...
(int o) complex_function (int arr[]) "pkg" "0.0.1" "complex";
@dispatch=WORKER
(int o) process (int i) "pkg" "0.0.1" [
  "set <<o>> [ pkg::process <<i>> ]"
];
pragma worktypedef foo_work;

@dispatch=foo_work
hello1 (string name) "turbine" "0.0" [
  "puts [ format {Hello %s} <<msg>> ]"
];

hello2 (string name) "turbine" "0.0" [
  "puts [ format {Hello %s} <<msg>> ]"
]

// Hello Foo will be printed on a foo_work worker
hello1("Foo");

// Hello Bar will be printed on a regular worker
hello2("Bar");
import files;

app (file out) cat (file inputs[]) {
  "/bin/cat" inputs @stdout=out
}

file joined <"joined.txt"> = cat(glob("*.txt"));
app (void signal) sleep (int secs) {
  "/bin/sleep" secs
}

foreach time in [1:5] {
  void signal = sleep(time);
  // Wait on output signal so that trace occurs after sleep
  wait(signal) {
    trace("Slept " + fromint(time));
  }
}
@dispatch=COASTER
app (file out) echo (string args[]) {
  "echo" args @stdout=out
}
export WORKER_MODE=local
export IPADDR=127.0.0.1
export SERVICE_PORT=53363
export JOBSPERNODE=4
export LOGDIR=/home/user/swift-logs
export WORKER_LOG_DIR=/home/user/swift-logs

coaster-service -nosec -port ${SERVICE_PORT}
export TURBINE_COASTER_CONFIG="jobManager=local,maxParallelTasks=4,coasterServiceURL=${IPADDR}:${SERVICE_PORT}"
pragma appexecdef COASTER "turbine" "0.8.0"
    "turbine::async_exec_coaster <<cmd>> <<args>> <<stage_in>> <<stage_out>> <<props>> <<success>> <<failure>>";
import io;
import python;

i = python("print(\"python works\")\nrepr(2+2)");
printf("i: %s", i);
python works
i: 4
import io;
import python;
import string;

global const string numpy = "from numpy import *\n\n";

typedef matrix string;

(matrix A) eye(int n)
{
  string command = sprintf("repr(eye(%i))", n);
  string code = numpy+command;
  matrix t = python(code);
  A = replace_all(t, "\n", "", 0);
}

(matrix R) add(matrix A1, matrix A2)
{
  string command = sprintf("repr(%s+%s)", A1, A2);
  string code = numpy+command;
  matrix t = python(code);
  R = replace_all(t, "\n", "", 0);
}

matrix A1 = eye(3);
matrix A2 = eye(3);
matrix sum = add(A1, A2);
printf("2*eye(3)=%s", sum);
2*eye(3)=array([[ 2.,  0.,  0.],
                [ 0.,  2.,  0.],
                [ 0.,  0.,  2.]])
import io;
import string;
import R;

global const string template =
"""
  x <- %i
  a <- x+100
  cat("the answer is: ", a, "\\n")
  a
""";

code = sprintf(template, 4);
s = R(code);
printf("the answer was: %i", s);
the answer is:  104
the answer was: 104
import io;
import julia;
import string;
import sys;

start = clock();
f =
"""
begin
 f(x) = begin
          sleep(1)
          x+1
        end
 f(%s)
end
""";
s1 = julia(sprintf(f, 1));
s2 = julia(sprintf(f, 2));
s3 = julia(sprintf(f, 3));
printf("julia results: %s %s %s", s1, s2, s3);
wait (s1, s2, s3) {
  printf("duration: %0.2f", clock()-start);
}
foreach i in [0:n-1] {
  @prio=i f(i);         // or
  int j = @prio=i f(i);
}
foreach i in [0:n-1] {
  location L = locationFromRank(i);
  @location=L f(i);
}
location L = location(rank, HARD|SOFT, RANK|NODE);
(string o) msg(string a, int b=0, float c=0.0) {
  o = "%s %i %f" % (a, b, c);
}
trace(msg("test"));
trace(msg("test", 1));
trace(msg("test", 1, 2.0));
trace(msg("test", y=2.0));
(int o) my_sum(int... vals) "my_pkg" "1.0" [
  "set <<o>> [ my_pkg::my_sum <<vals>> ]"
];
import io;

() print_num(int x) {
  printf("int: %i", x);
}

() print_num(float x) {
  printf("float: %0.2f", x);
}

print_num(1);
print_num(3.14);
# No optimizations at all (not recommended)
stc -O0 example.swift example.tcl

# Basic optimizations (not recommended)
stc -O1 example.swift example.tcl

# Standard optimizations (recommended)
stc example.swift example.tcl
# OR
stc -O2 example.swift example.tcl

# All optimizations (also recommended)
stc -O3 example.swift example.tcl
stc -h
turbine <turbine arguments> <program.tcl> <program arguments>
export TURBINE_SRAND=$( date +%s )
turbine ...
cd c-utils
./configure ...
make install

cd ../lb
./configure ...
make install

cd ../turbine
./configure ...
make install

cd ../stc
ant install -Ddist.dir=... -Dturbine.home=...
EXM_CUSTOM_MPI=1
MPI_INCLUDE=/path/to/mpi.h/include
MPI_LIB_DIR=/path/to/mpi_lib/lib
MPI_LIB_NAME=funny.mpi.a
 --enable-custom --with-mpi-include=/path/to/mpi.h/include
                 --with-mpi-lib-dir=/path/to/mpi_lib/lib
                 --with-mpi-lib-name=funny.mpi.a
stc my.swift
mkstatic.tcl my.manifest -c my_main.c
gcc -static script_main.c file1.o file2.o -L/path/to/lib/dir -lsomething ...
export VALGRIND="valgrind --suppressions=$HOME/turbine/turbine.supp"
turbine program.tcl
$ export GDB_RANK=0
$ turbine program.tcl
Waiting for gdb: rank: 0 pid: 23274
...

Swift/T Guide

1. Support

2. Installation

2.1. Installation of Prerequisites

2.2. Installation of Swift/T from Source

3. Usage

4. Program structure

5. Comments

6. Modules

7. Variable declarations

8. Dataflow evaluation

9. Composite functions

10. Types

10.1. Primitive types

10.2. Files

10.3. Blobs

10.4. Arrays

10.5. Nested arrays

10.6. Structs

10.7. Defining new types

10.8. Global Variables

11. Control structures

11.1. Conditionals

11.1.1. If statement

11.1.2. Switch statement

11.2. Iteration

11.2.1. Foreach loop

11.2.2. For loop

11.3. Explicit data-dependent execution

11.4. Scoping blocks

12. Operators

12.1. Numeric operators

12.2. Boolean operators

12.3. String operators

13. Standard library

13.1. General

13.2. Type conversion

13.3. Output

13.4. String functions

13.5. Math

13.6. System

13.6.1. Command line

13.6.2. Debugging

13.6.3. Turbine information

13.7. Files

13.8. Blobs

13.9. Location

14. Defining leaf functions

14.1. Swift extension functions

14.1.1. Simple Tcl fragment example

14.1.2. Simple Tcl package example

14.1.3. Swift/Tcl data type mapping

14.1.4. Calling native libraries from Swift

14.1.5. Writing custom Tcl interfaces

14.2. Dispatch and work types

14.2.1. Custom work types

14.3. App functions

14.4. Remote job execution with Coasters

14.5. Custom App Executors (Advanced)

14.6. External scripting support

14.6.1. Calling Python

14.6.2. Calling R

14.6.3. Calling Julia

15. More about functions

15.1. Function call annotations

15.1.1. Priority

15.1.2. Location

Hostmap

15.2. Advanced function topics

15.2.1. Optional and keyword arguments to functions

15.2.2. Variable-length argument lists

15.2.3. Function overloading

15.2.4. Union argument types and type variables

16. Optimizations

17. Running in Turbine

17.1. Architecture

17.2. Concurrency

17.3. Invocation

17.4. Build configuration

17.4.1. Build troubleshooting