layers -> xlayers
This commit is contained in:
parent
eabdd21d97
commit
badb6cf51d
@ -43,5 +43,5 @@ jobs:
|
||||
echo "Show what we have here:"
|
||||
ls
|
||||
python --version
|
||||
python -m pytest ./tests -s
|
||||
python -m pytest ./tests/test_basic_space.py -s
|
||||
shell: bash
|
32
.github/workflows/super_model_test.yml
vendored
Normal file
32
.github/workflows/super_model_test.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
name: Run Python Tests for Super Model
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04, macos-latest]
|
||||
python-version: [3.6, 3.7, 3.8, 3.9]
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Test Super Model
|
||||
run: |
|
||||
python -m pip install pytest numpy
|
||||
python -m pip install torch torchvision torchaudio
|
||||
python -m pytest ./tests/test_super_model.py -s
|
||||
shell: bash
|
@ -1,5 +0,0 @@
|
||||
from .drop import DropBlock2d, DropPath
|
||||
from .mlp import MLP
|
||||
from .weight_init import trunc_normal_
|
||||
|
||||
from .positional_embedding import PositionalEncoder
|
@ -7,6 +7,8 @@
|
||||
from .basic_space import Categorical
|
||||
from .basic_space import Continuous
|
||||
from .basic_space import Integer
|
||||
from .basic_space import Space
|
||||
from .basic_space import VirtualNode
|
||||
from .basic_op import has_categorical
|
||||
from .basic_op import has_continuous
|
||||
from .basic_op import get_min
|
||||
|
@ -7,6 +7,7 @@ import math
|
||||
import copy
|
||||
import random
|
||||
import numpy as np
|
||||
from collections import OrderedDict
|
||||
|
||||
from typing import Optional
|
||||
|
||||
@ -44,6 +45,32 @@ class Space(metaclass=abc.ABCMeta):
|
||||
return copy.deepcopy(self)
|
||||
|
||||
|
||||
class VirtualNode(Space):
|
||||
"""For a nested search space, we represent it as a tree structure.
|
||||
|
||||
For example,
|
||||
"""
|
||||
|
||||
def __init__(self, id=None, value=None):
|
||||
self._id = id
|
||||
self._value = value
|
||||
self._attributes = OrderedDict()
|
||||
|
||||
def has(self, x):
|
||||
for key, value in self._attributes.items():
|
||||
if isinstance(value, Space) and value.has(x):
|
||||
return True
|
||||
return False
|
||||
|
||||
def __repr__(self):
|
||||
strs = [self.__class__.__name__ + "("]
|
||||
indent = " " * 4
|
||||
for key, value in self._attributes.items():
|
||||
strs.append(indent + strs(value))
|
||||
strs.append(")")
|
||||
return "\n".join(strs)
|
||||
|
||||
|
||||
class Categorical(Space):
|
||||
"""A space contains the categorical values.
|
||||
It can be a nested space, which means that the candidate in this space can also be a search space.
|
||||
|
@ -12,7 +12,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
import layers as xlayers
|
||||
import xlayers
|
||||
|
||||
|
||||
DEFAULT_NET_CONFIG = dict(
|
||||
|
11
lib/xlayers/__init__.py
Normal file
11
lib/xlayers/__init__.py
Normal file
@ -0,0 +1,11 @@
|
||||
#####################################################
|
||||
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 #
|
||||
#####################################################
|
||||
# This file is expected to be self-contained, expect
|
||||
# for importing from spaces to include search space.
|
||||
#####################################################
|
||||
from .drop import DropBlock2d, DropPath
|
||||
from .mlp import MLP
|
||||
from .weight_init import trunc_normal_
|
||||
|
||||
from .positional_embedding import PositionalEncoder
|
@ -1,16 +1,15 @@
|
||||
#####################################################
|
||||
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.03 #
|
||||
#####################################################
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn.parameter import Parameter
|
||||
from torch import Tensor
|
||||
|
||||
import math
|
||||
from typing import Optional, Union
|
||||
|
||||
import spaces
|
||||
from layers.super_module import SuperModule
|
||||
from layers.super_module import SuperRunType
|
||||
from .super_module import SuperModule
|
||||
from .super_module import SuperRunMode
|
||||
|
||||
IntSpaceType = Union[int, spaces.Integer, spaces.Categorical]
|
||||
BoolSpaceType = Union[bool, spaces.Categorical]
|
||||
@ -32,11 +31,11 @@ class SuperLinear(SuperModule):
|
||||
self._out_features = out_features
|
||||
self._bias = bias
|
||||
|
||||
self._super_weight = Parameter(
|
||||
self._super_weight = torch.nn.Parameter(
|
||||
torch.Tensor(self.out_features, self.in_features)
|
||||
)
|
||||
if bias:
|
||||
self._super_bias = Parameter(torch.Tensor(self.out_features))
|
||||
if self.bias:
|
||||
self._super_bias = torch.nn.Parameter(torch.Tensor(self.out_features))
|
||||
else:
|
||||
self.register_parameter("_super_bias", None)
|
||||
self.reset_parameters()
|
||||
@ -53,6 +52,9 @@ class SuperLinear(SuperModule):
|
||||
def bias(self):
|
||||
return spaces.has_categorical(self._bias, True)
|
||||
|
||||
def abstract_search_space(self):
|
||||
print('-')
|
||||
|
||||
def reset_parameters(self) -> None:
|
||||
nn.init.kaiming_uniform_(self._super_weight, a=math.sqrt(5))
|
||||
if self.bias:
|
||||
@ -60,7 +62,7 @@ class SuperLinear(SuperModule):
|
||||
bound = 1 / math.sqrt(fan_in)
|
||||
nn.init.uniform_(self._super_bias, -bound, bound)
|
||||
|
||||
def forward_raw(self, input: Tensor) -> Tensor:
|
||||
def forward_raw(self, input: torch.Tensor) -> torch.Tensor:
|
||||
return F.linear(input, self._super_weight, self._super_bias)
|
||||
|
||||
def extra_repr(self) -> str:
|
@ -14,12 +14,12 @@ class SuperRunMode(Enum):
|
||||
Default = "fullmodel"
|
||||
|
||||
|
||||
class SuperModule(abc.ABCMeta, nn.Module):
|
||||
class SuperModule(abc.ABC, nn.Module):
|
||||
"""This class equips the nn.Module class with the ability to apply AutoDL."""
|
||||
|
||||
def __init__(self):
|
||||
super(SuperModule, self).__init__()
|
||||
self._super_run_type = SuperRunMode.default
|
||||
self._super_run_type = SuperRunMode.Default
|
||||
|
||||
@abc.abstractmethod
|
||||
def abstract_search_space(self):
|
@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -31,29 +31,42 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"SuperRunType.FullModel\n",
|
||||
"SuperRunType.FullModel\n",
|
||||
"True\n",
|
||||
"True\n"
|
||||
"ename": "AttributeError",
|
||||
"evalue": "default",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m~/Desktop/XAutoDL/notebooks/spaces\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mout_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCategorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m12\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m24\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m36\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCategorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSuperLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m~/Desktop/XAutoDL/lib/layers/super_mlp.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, in_features, out_features, bias)\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mBoolSpaceType\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m ) -> None:\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSuperLinear\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;31m# the raw input args\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m~/Desktop/XAutoDL/lib/layers/super_module.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSuperModule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_super_run_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSuperRunMode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefault\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mabc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m~/anaconda3/lib/python3.8/enum.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(cls, name)\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_member_map_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 340\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;31mAttributeError\u001b[0m: default"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Test the Linear layer\n",
|
||||
"import spaces\n",
|
||||
"from layers.super_core import SuperLinear\n",
|
||||
"from layers.super_module import SuperRunMode\n",
|
||||
"\n",
|
||||
"print(SuperRunMode.Default)\n",
|
||||
"print(SuperRunMode.FullModel)\n",
|
||||
"print(SuperRunMode.Default == SuperRunMode.FullModel)\n",
|
||||
"print(SuperRunMode.FullModel == SuperRunMode.FullModel)"
|
||||
"out_features = spaces.Categorical(12, 24, 36)\n",
|
||||
"bias = spaces.Categorical(True, False)\n",
|
||||
"model = SuperLinear(10, out_features, bias=bias)\n",
|
||||
"print(model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
28
tests/test_super_model.py
Normal file
28
tests/test_super_model.py
Normal file
@ -0,0 +1,28 @@
|
||||
#####################################################
|
||||
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.03 #
|
||||
#####################################################
|
||||
# pytest ./tests/test_super_model.py -s #
|
||||
#####################################################
|
||||
import sys, random
|
||||
import unittest
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
|
||||
lib_dir = (Path(__file__).parent / ".." / "lib").resolve()
|
||||
print("library path: {:}".format(lib_dir))
|
||||
if str(lib_dir) not in sys.path:
|
||||
sys.path.insert(0, str(lib_dir))
|
||||
|
||||
import torch
|
||||
from xlayers import super_core
|
||||
import spaces
|
||||
|
||||
|
||||
class TestSuperLinear(unittest.TestCase):
|
||||
"""Test the super linear."""
|
||||
|
||||
def test_super_linear(self):
|
||||
out_features = spaces.Categorical(12, 24, 36)
|
||||
bias = spaces.Categorical(True, False)
|
||||
model = super_core.SuperLinear(10, out_features, bias=bias)
|
||||
print(model)
|
Loading…
Reference in New Issue
Block a user