In [1]:

    

import multiprocessing as mp
import sh


    

In [2]:

    

n_process = 4
n_lines = 1000
file_name = '/tmp/input.txt'


    

In [3]:

    

# create a new file
with open(file_name, 'w') as f:
    for i in range(n_lines):
        f.write('%s\n' % i)


    

In [4]:

    

def apply_double(file_name, i_start, i_final, output):
    i_start += 1
    i_final += 1
    
    result = sh.sed('-n', '%s,%sp' % (i_start, i_final), file_name)
    
    for v in result.split('\n'):
        if not v == '':
            output.put(int(v)*2)


    

In [5]:

    

output = mp.Queue()

chunk = n_lines / n_process

processes = [
    mp.Process(
        target=apply_double, args=(
            file_name, int(i*chunk), int((i+1) * chunk
        ), output)
    ) for i in range(4)
]

# Run processes
for p in processes:
    p.start()

# Exit the completed processes
for p in processes:
    p.join()

# Get process results from the output queue
results = []

for _ in range(n_lines):
    results += [output.get()]

print(results)


    

    




[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 160, 162, 164, 166, 168, 170, 172, 174, 176, 178, 180, 182, 184, 186, 188, 190, 192, 194, 196, 198, 200, 202, 204, 206, 208, 210, 212, 214, 216, 218, 220, 222, 224, 226, 228, 230, 232, 234, 236, 238, 240, 242, 244, 246, 248, 250, 252, 254, 256, 258, 260, 262, 264, 266, 268, 270, 272, 274, 276, 278, 280, 282, 284, 286, 288, 290, 292, 294, 296, 298, 300, 302, 304, 306, 308, 310, 312, 314, 316, 318, 320, 322, 324, 326, 328, 330, 332, 334, 336, 338, 340, 342, 344, 346, 348, 350, 352, 354, 356, 358, 360, 362, 364, 366, 368, 370, 372, 374, 376, 378, 380, 382, 384, 386, 388, 390, 392, 394, 396, 398, 400, 402, 404, 406, 408, 410, 412, 414, 416, 418, 420, 422, 424, 426, 428, 430, 432, 434, 436, 438, 440, 442, 444, 446, 448, 450, 1500, 1502, 1000, 500, 1002, 1004, 1006, 1008, 1010, 502, 1012, 1014, 504, 1504, 1016, 506, 1018, 508, 1020, 510, 1022, 512, 1024, 514, 1026, 516, 1028, 518, 1030, 520, 1032, 522, 1034, 524, 1036, 526, 1038, 528, 1040, 530, 1042, 532, 1044, 534, 1046, 536, 1048, 538, 1050, 540, 1052, 542, 1054, 544, 1056, 546, 1058, 548, 1060, 550, 1062, 552, 1064, 554, 1066, 556, 1068, 1070, 558, 1072, 560, 1074, 562, 1076, 564, 1078, 566, 1080, 568, 1082, 570, 1084, 572, 1086, 1088, 1090, 574, 1092, 576, 452, 1094, 1096, 1098, 1100, 578, 454, 1102, 1104, 1106, 1108, 580, 456, 1110, 1112, 1114, 1116, 582, 458, 1118, 1120, 1122, 1124, 584, 460, 1126, 1128, 1130, 1132, 586, 1134, 462, 588, 1136, 1138, 1140, 1142, 464, 590, 1144, 1146, 1148, 1150, 466, 468, 470, 1506, 1508, 1510, 592, 594, 596, 1152, 1154, 1156, 472, 474, 476, 1512, 1158, 478, 1514, 1516, 480, 482, 1518, 1520, 484, 486, 488, 1160, 1162, 1164, 598, 600, 602, 1522, 1524, 604, 606, 608, 1166, 1168, 1170, 490, 492, 494, 496, 498, 500, 1526, 1172, 1528, 1174, 1176, 1178, 1180, 1530, 610, 1532, 1534, 612, 614, 1536, 1182, 1538, 1184, 1540, 1186, 1188, 1542, 616, 1544, 618, 1546, 620, 1548, 622, 1550, 624, 1552, 626, 1554, 628, 1556, 630, 1558, 632, 1560, 634, 1562, 636, 1564, 638, 1566, 640, 1568, 642, 1570, 644, 1572, 646, 1574, 648, 1576, 650, 1578, 652, 654, 656, 658, 660, 662, 664, 666, 668, 670, 672, 1580, 674, 1582, 676, 1584, 678, 1586, 680, 1588, 682, 1590, 684, 1592, 686, 1594, 688, 1596, 690, 1598, 692, 1600, 694, 1602, 696, 1604, 698, 1606, 700, 702, 1608, 704, 1610, 706, 1612, 708, 710, 712, 1614, 714, 1616, 1190, 1618, 1192, 1620, 1194, 1196, 1622, 1198, 1624, 1200, 1626, 1202, 1628, 1204, 1630, 1206, 1632, 1208, 1634, 1210, 1636, 1212, 1638, 1214, 1640, 1216, 1642, 1218, 1644, 1220, 1646, 1648, 1650, 1652, 1654, 1656, 1658, 1660, 1222, 1662, 1224, 1664, 1226, 1228, 1666, 1230, 1668, 1232, 1670, 1234, 1236, 1672, 1238, 1674, 1240, 1676, 1242, 1678, 1244, 1680, 1246, 1682, 1248, 1684, 1250, 1686, 1252, 1688, 1254, 1256, 1690, 1258, 1692, 1260, 1694, 1262, 1696, 1264, 1698, 1266, 1700, 1268, 1702, 1270, 1704, 1272, 1706, 1274, 1708, 1276, 1710, 1278, 1712, 1280, 1714, 1716, 1718, 1720, 1722, 1724, 1726, 1728, 1730, 1732, 1734, 1736, 1738, 1740, 1742, 1744, 1746, 1748, 1750, 1752, 1754, 1756, 1758, 1760, 1762, 1764, 1766, 1768, 1770, 1772, 1774, 1776, 1778, 1780, 1782, 1784, 1786, 1788, 1790, 1792, 1794, 1796, 1798, 1800, 1802, 1804, 1806, 1808, 1810, 1812, 1814, 1816, 1818, 1820, 1822, 1824, 1826, 1828, 1830, 1832, 1834, 1836, 1838, 1840, 1842, 1844, 1846, 1848, 1850, 1852, 1854, 1856, 1858, 1860, 1862, 1864, 1866, 1868, 1870, 1872, 1874, 1876, 1878, 1880, 1882, 1884, 1886, 1888, 1890, 1892, 1894, 1896, 1898, 1900, 1902, 1904, 1906, 1908, 1910, 1912, 1914, 1916, 1918, 1920, 1922, 1924, 1926, 1928, 1930, 1932, 1934, 1936, 1938, 1940, 1942, 1944, 1946, 1948, 1950, 1952, 1954, 1956, 1958, 1960, 1962, 1964, 1966, 1968, 1970, 1972, 1974, 1976, 1978, 1980, 1982, 1984, 1986, 1988, 1990, 1992, 1994, 1996, 1998, 716, 718, 720, 722, 724, 726, 728, 730, 732, 734, 736, 738, 740, 742, 744, 746, 748, 750, 752, 754, 756, 758, 760, 762, 764, 766, 768, 770, 772, 774, 776, 778, 780, 782, 1282, 784, 786, 1284, 788, 1286, 790, 1288, 792, 1290, 794, 1292, 796, 1294, 798, 1296, 800, 1298, 802, 1300, 804, 1302, 806, 1304, 808, 1306, 810, 1308, 812, 1310, 814, 1312, 816, 1314, 818, 1316, 820, 1318, 822, 1320, 824, 1322, 826, 1324, 828, 1326, 830, 1328, 832, 1330, 834, 1332, 836, 1334, 838, 1336, 840, 1338, 842, 1340, 844, 1342, 846, 1344, 848, 1346, 850, 1348, 852, 854, 856, 858, 860, 862, 1350, 864, 1352, 866, 1354, 868, 1356, 870, 1358, 872, 1360, 874, 1362, 876, 1364, 878, 1366, 880, 1368, 882, 1370, 884, 1372, 886, 1374, 888, 1376, 890, 1378, 892, 1380, 894, 1382, 896, 1384, 898, 900, 902, 1386, 904, 1388, 906, 1390, 908, 1392, 910, 1394, 912, 1396, 914, 1398, 916, 1400, 918, 1402, 1404, 1406, 1408, 920, 1410, 922, 1412, 924, 1414, 926, 1416, 928, 1418, 930, 1420, 932, 1422, 934, 1424, 936, 1426, 938, 1428, 940, 1430, 942, 1432, 944, 1434, 946, 1436, 948, 1438, 950, 1440, 952, 1442, 954, 1444, 956, 1446, 958, 1448, 960, 1450, 962, 1452, 964, 1454, 966, 1456, 968, 1458, 970, 1460, 972, 1462, 974, 1464, 1466, 1468, 1470, 1472, 1474, 1476, 1478, 1480, 1482, 1484, 1486, 1488, 1490, 1492, 1494, 1496, 1498, 1500, 976, 978, 980, 982, 984, 986, 988, 990, 992, 994]