[Bug?] Inserting a layer into a layout restores previously clipped shapes

edited April 2023 in Python scripting

Hi,
i'm trying to clip certain areas (cells, layers, boxes) of a layout.
For this purpose i'm mainly using Layout.clip_into - I'm purposefully avoiding multi_clip_into, because I might have similar clipping areas but need the cells the be distinct at a later point in time.

This method assumes the presence of the layers that ought to be clipped in the new layout, therefore I'm copying the layers beforehand.
However, if I do not copy all layers and insert a new layer after performing the clip it get inserted at the lowest index and restores the shapes.

Here's an example script showing the workflow and the problem.
First the data used to test for this problem:

import pya
from typing import List, Tuple, Union

def one(layout):
    return 1 / layout.dbu


def layout_with_top_cell(dbu=None):
    if dbu is None:
        dbu = 0.001

    layout = pya.Layout()
    layout.dbu = dbu

    layout.add_cell("top")

    return layout

# this generates a layout with boxes along the diagonal (if not specified otherwise)
def multi_box_layout(dbu=0.001, new_cells=True, new_layer=True, n_boxes=10, box_size=1,
                     start_x=0, start_y=0, offset_x=1, offset_y=1):
    layout = layout_with_top_cell(dbu)
    top: pya.Cell = layout.cell(0)

    side_length = box_size * one(layout)
    left = start_x * one(layout)
    lower = start_y * one(layout)

    for i in range(n_boxes):
        if new_layer:
            layer_index = layout.insert_layer(pya.LayerInfo(i, i))
        else:
            layer_index = layout.layer(0, 0)

        if new_cells:
            cell_index = layout.add_cell(f"box_{i}")
            inst = pya.CellInstArray(cell_index, pya.Trans())
            top.insert(inst)
        else:
            cell_index = top.cell_index()

        box = pya.Box(left, lower, left + side_length, lower + side_length)
        layout.cell(cell_index).shapes(layer_index).insert(box)

        left += offset_x * one(layout)
        lower += offset_y * one(layout)

    return layout

The function multi_box_layout will generate a layout for use which has boxes of side length 1 across the diagonal, each in a new cell and in a new layer.
We now need the function to clip this layout at certain positions and return the clipped layout:

def get_index_for_layer_info(layout: pya.Layout, query_layer_infos: List[pya.LayerInfo]) -> List[Union[int, None]]:
    results = [None] * len(query_layer_infos)

    for layer_index in layout.layer_indices():
        layer_info: pya.LayerInfo = layout.get_info(layer_index)
        for qindex, qlayer_info in enumerate(query_layer_infos):
            # https://www.klayout.de/doc-qt5/code/class_LayerInfo.html#m_is_equivalent?
            if qlayer_info.is_equivalent(layer_info) and results[qindex] is None:
                results[qindex] = layer_index

    return results

def copy_layers_by_layer_info(layout_old: pya.Layout, layout_new: pya.Layout, clips: List[pya.LayerInfo]) -> pya.Layout:
    layer_indices = get_index_for_layer_info(layout_old, clips)
    layer_indices = set(i for i in layer_indices if i is not None)

    for layer_index in layer_indices:
        layout_new.insert_layer_at(layer_index, layout_old.get_info(layer_index))
    return layout_new

def create_clipped_layout(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                          clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
    layout_new = pya.Layout()
    layout_new.dbu = layout_input.dbu
    copy_layers_by_layer_info(
        layout_input, layout_new, layers
    )  # layers need to be copied for clip_into

    cell_indices = []
    for clip in clips:
        cell_index = layout_input.clip_into(
            layout_input.top_cell().cell_index(), layout_new, clip
        )
        cell_indices.append(cell_index)

    return layout_new, cell_indices

The function create_clipped_layout does the heavy lifting here and uses the two other functions as utility. It starts by copying the specified layers from the input layout to the newly generated layout and afterwards performs the clips into the new layout.
Now, to test test its functionality:

layout = multi_box_layout()
one_ = one(layout)
test_shapes = [0, 2]
test_boxes = [pya.Box(i * one_, i * one_, (i +1) * one_, (i + 1) * one_) for i in test_shapes]
test_layers = [pya.LayerInfo(i, i) for i in test_shapes]
test_region = sum([pya.Region(i) for i in test_boxes], start=pya.Region())

# we try to clip the whole layout (big box) but only certain layers
# because each box has its own layer, we should end up only with the boxes given by the layers
complete_box = pya.Box(0, 0, 10 * one_, 10 * one_)
clipped_layout, cell_indices = create_clipped_layout(layout, test_layers, [complete_box])
actual_shapes = pya.Region()
for top_cell in clipped_layout.top_cells():
    for layer_index in clipped_layout.layer_indices():
        actual_shapes += pya.Region(top_cell.begin_shapes_rec(layer_index))
assert (actual_shapes - test_region).is_empty(), "shapes do not match"

This works like expected and the we only have shapes at box positions 0 and 2 along the diagonal. (these indices specify the lower left coordinate of the box).
However, if we now insert a new layer (it does not matter whether we use a LayerInfo that was previously present or not), we run into problems.

# now we insert a new layer (which should be empty)
clipped_layout.insert_layer(pya.LayerInfo(999,999))

# but its not?
actual_shapes = pya.Region()
for top_cell in clipped_layout.top_cells():
    for layer_index in clipped_layout.layer_indices():
        actual_shapes += pya.Region(top_cell.begin_shapes_rec(layer_index))
assert (actual_shapes - test_region).is_empty(), "shapes do not match"

Any ideas where the error lies? How can this use case be implemented in a better way?
Thanks.

Comments

  • edited April 2023

    I observed more things.
    If we save the layout before inserting the new layer, the shapes do indeed match.

    clipped_layout.write("test.gds")
    clipped_layout = pya.Layout()
    clipped_layout.read("test.gds")
    

    This is also the case if we "copy" the layout as in:

    def copy_layout(layout: pya.Layout) -> pya.Layout:
        combined_layout = pya.Layout()
        combined_layout.dbu = layout.dbu
        try:
            cell = combined_layout.create_cell(layout.top_cell().name)
            cell.copy_tree(layout.top_cell())
        except RuntimeError:  # multiple top cells
            for top_cell in layout.top_cells():
                cell: pya.Cell = combined_layout.create_cell(top_cell.name)
                cell.copy_tree(top_cell)
        return combined_layout
    
    clipped_layout = copy_layout(clipped_layout)
    

    (but i think this is a hacky workaround)
    However, merging the copy procedure into the clipping procedure does not work.

    def create_clipped_layout_with_copy(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                              clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
        copy_layers_by_layer_info(
            layout_input, layout_new, layers
        )  # layers need to be copied for clip_into
    
        cell_indices = []
        for index, clip in enumerate(clips):
            cell_index = layout_input.clip(
                layout_input.top_cell().cell_index(), clip
            )
            new_cell: pya.Cell = layout_new.create_cell(str(index))
            new_cell.move_tree(layout_input.cell(cell_index))
            #layout_input.delete_cell(cell_index)
            cell_indices.append(new_cell.cell_index())
        return layout_new, cell_indices
    

    Nor does deleting empty cells.

    def create_clipped_layout_del_cells(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                              clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
        copy_layers_by_layer_info(
            layout_input, layout_new, layers
        )  # layers need to be copied for clip_into
    
        cell_indices = []
        for clip in clips:
            cell_index = layout_input.clip_into(
                layout_input.top_cell().cell_index(), layout_new, clip
            )
            cell_indices.append(cell_index)
    
        layout_new.start_changes()
        layout_new.delete_cells([ c.cell_index() for c in layout_new.each_cell() if c.is_empty() ])
        layout_new.end_changes()
        return layout_new, cell_indices
    

    I'm grateful for any help to improve the create_clipped_layout method, such that i can specify a layout, layers within the layout and boxes, and that the returned layout corresponds to only these given parameters and can withstand layer insertion.

  • edited April 2023

    I somwhat solved this issue by creating the layers and shape containers by hand. However, I'm no longer able to reconstruct the original cell tree, because I'm using the Region API. This could be achieved by performing the clipping hierarchically as well. Moreover, this function is really slow, hence I will stick to the hacky workaround with copying the layout at the end.

    def create_clipped_layout_region(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                              clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
    
        for layer_info, layer_index in zip(layers, get_index_for_layer_info(layout_input, layers)):
            if layer_index is not None:
                layout_new.insert_layer(layer_info)
        layermap: pya.LayerMapping = pya.LayerMapping().create(layout_new, layout_input)
    
        cell_indices = []
        for index, clip in enumerate(clips):
            clip_cell: pya.Cell = layout_new.create_cell(str(index))
            cell_indices.append(clip_cell.cell_index())
    
            clip_region = pya.Region(clip)
            for layer_from, layer_to in layermap.table().items():
                shapes = pya.Region(layout_input.top_cell().begin_shapes_rec(layer_from))
                clipped_shapes = clip_region & shapes
                clip_cell.shapes(layer_to).insert(clipped_shapes)
    
        return layout_new, cell_indices
    
    clipped_layout, cell_indices = create_clipped_layout_region(layout, test_layers, [complete_box])
    clipped_layout.insert_layer(pya.LayerInfo(999,999))
    test_layout(clipped_layout)
    
  • edited April 2023

    Hi @Peter123,

    thanks a lot for this nicely prepared test case.

    I need to say, that the "clip_into" function is not confining the clip to certain layers. It will always clip all layers. And it will always assume that all layers are present in the target layout. So right now, the proper way of doing the clip is:

    target_layout = pya.Layout()
    # fill in the layers
    for layer_index in source_layout.layer_indexes():
            target_layout.insert_layer_at(layer_index, source_layout.get_info(layer_index))
    
    # and then do the clip
    source_layout.clip_into(from_cell, target_layout, clip_box)
    

    Without this, "clip_into" force-creates the layers without properly configuring the layout and that makes the assertion fail: as you correctly observed, new layers are created with layer indexes already populated by "clip_into". Call this a bug if you like.

    So what is the solution?

    A straightforward, but somehow clumsy solution is to delete the layers not needed after the clip:

    # NOTE: clips argument no longer needed
    def copy_layers_by_layer_info(layout_old: pya.Layout, layout_new: pya.Layout) -> pya.Layout:
        for layer_index in layout_old.layer_indexes():
            layout_new.insert_layer_at(layer_index, layout_old.get_info(layer_index))
        return layout_new
    
    def delete_layers_not_required(layout_new: pya.Layout, clips: List[pya.LayerInfo]):
        for layer_index in layout_new.layer_indexes():
            if not layout_new.get_info(layer_index) in clips:
                layout_new.delete_layer(layer_index)
    
    def create_clipped_layout(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                              clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
        copy_layers_by_layer_info(
            layout_input, layout_new
        )  # layers need to be copied for clip_into
    
        cell_indices = []
        for clip in clips:
            cell_index = layout_input.clip_into(
                layout_input.top_cell().cell_index(), layout_new, clip
            )
            cell_indices.append(cell_index)
    
        delete_layers_not_required(
            layout_new, layers
        )
    
        return layout_new, cell_indices
    

    With this, the first test script passes.

    A more efficient solution is to avoid clip_into and do the clip in the source layout, then copy. "move_tree" (or "copy_tree") will also copy all layers, so not much is gained. Instead you have to use "copy_tree_shapes" which is powerful, but needs to be configured. In this case, we need to tell it we want the hierarchy to be copied and give it a layer map that only selects the requested layers.

    The implementation of create_clipped_layout_with_copy is then:

    def create_clipped_layout_with_copy(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                                        clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
    
        cell_indices = []
        for index, clip in enumerate(clips):
    
            cell_index = layout_input.clip(
                layout_input.top_cell().cell_index(), clip
            )
    
            new_cell = layout_new.create_cell(str(index))
    
            # prepare a target hierarchy
            cm = pya.CellMapping()
            cm.for_single_cell_full(new_cell, layout_input.cell(cell_index))
    
            # prepare the layer map
            lm = pya.LayerMapping()
            for layer in layers:
                lm.map(layout_input.layer(layer), layout_new.layer(layer))
    
            new_cell.copy_tree_shapes(layout_input.cell(cell_index), cm, lm)
    
            cell_indices.append(new_cell.cell_index())
    
        return layout_new, cell_indices
    

    If you plan to separate the cells anyway through copy, there is nothing wrong with "multi_clip" - on the contrary: it may be more efficient. With that, the solution looks like that:

    def create_clipped_layout_multi_with_copy(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                                              clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
    
        old_cell_indices = layout_input.multi_clip(
            layout_input.top_cell().cell_index(), clips
        )
    
        cell_indices = []
        for index, cell_index in enumerate(old_cell_indices):
    
            new_cell = layout_new.create_cell(str(index))
    
            # prepare a target hierarchy
            cm = pya.CellMapping()
            cm.for_single_cell_full(new_cell, layout_input.cell(cell_index))
    
            # prepare the layer map
            lm = pya.LayerMapping()
            for layer in layers:
                lm.map(layout_input.layer(layer), layout_new.layer(layer))
    
            new_cell.copy_tree_shapes(layout_input.cell(cell_index), cm, lm)
    
            cell_indices.append(new_cell.cell_index())
    
        return layout_new, cell_indices
    

    The versions above pass your test in my case.

    Kind regards,

    Matthias

  • edited April 2023

    @Matthias, thanks a lot for the wonderful response. I tried it, and it indeed passes my testcase.
    However i also try to keep the input layout in its original form, therefore i have to delete (delete_cell_rec) the clipped top cells at the end.
    The problem now is, that cells, that are clipped and use a common cellInst, also have this cellInst deleted in the clips, therefore only the first instance keeps the shapes. I tried solving it, by not using multi_clip_into, but didnt succeed.

    Test Data (this time a layout with 4 unit-boxes at (0,0), (1,1), (1,0), (2,1), where the two last boxes are shared amongst cells:

    import pya
    from typing import List, Tuple, Union
    
    def multi_box_layout2():
        layout = pya.Layout()
        layout.dbu = 1
    
        layout.add_cell("top")
        top: pya.Cell = layout.cell(0)
    
        layer_index = layout.insert_layer(pya.LayerInfo(0, 0))
    
        cell_index_1 = layout.add_cell("1")
        inst = pya.CellInstArray(cell_index_1, pya.Trans())
        top.insert(inst)
        box = pya.Box(0, 0, 1, 1)
        layout.cell(cell_index_1).shapes(layer_index).insert(box)
    
        cell_index_2 = layout.add_cell("2")
        inst = pya.CellInstArray(cell_index_2, pya.Trans())
        top.insert(inst)
        box = pya.Box(1, 1, 2, 2)
        layout.cell(cell_index_2).shapes(layer_index).insert(box)
    
        cell_index_3 = layout.add_cell("3")
        box = pya.Box(0,0,1,1)
        layout.cell(cell_index_3).shapes(layer_index).insert(box)
    
        inst = pya.CellInstArray(cell_index_3, pya.Trans(2,1))
        layout.cell(cell_index_1).insert(inst)
        inst = pya.CellInstArray(cell_index_3, pya.Trans(1,0))
        layout.cell(cell_index_2).insert(inst)
    
        return layout
    
    clip_boxes = [
        pya.Box(0,0,2,1),
        pya.Box(1,1,2,2),
    ]
    clip_layers = [pya.LayerInfo(0,0)]
    test_region = sum([pya.Region(i) for i in clip_boxes], start=pya.Region())
    
    def test_layout(layout_to_test, layout_input):
        actual_shapes = pya.Region()
        for top_cell in layout_to_test.top_cells():
            for layer_index in layout_to_test.layer_indices():
                actual_shapes += pya.Region(top_cell.begin_shapes_rec(layer_index))
        assert (actual_shapes ^ test_region).is_empty(), f"shapes in clip do not match {actual_shapes} - {test_region}"
    
        layout_input_validation = multi_box_layout2()
        layout_input_region = pya.Region(layout_input.top_cell().begin_shapes_rec(0))
        layout_input_region_val = pya.Region(layout_input_validation.top_cell().begin_shapes_rec(0))
        assert (layout_input_region ^ layout_input_region_val).is_empty(), f"shapes in original do not match {actual_shapes} - {test_region}"
    

    Clip Method:

    def create_clipped_layout(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                                              clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
    
        old_cell_indices = layout_input.multi_clip(
            layout_input.top_cell().cell_index(), clips
        )
    
        cell_indices = []
        for index, cell_index in enumerate(old_cell_indices):
    
            new_cell = layout_new.create_cell(str(index))
    
            # prepare a target hierarchy
            cm = pya.CellMapping()
            cm.for_single_cell_full(new_cell, layout_input.cell(cell_index))
    
            # prepare the layer map
            lm = pya.LayerMapping()
            for layer in layers:
                lm.map(layout_input.layer(layer), layout_new.layer(layer))
    
            new_cell.copy_tree_shapes(layout_input.cell(cell_index), cm, lm)
            cell_indices.append(new_cell.cell_index())
    
        for cell_index in old_cell_indices:
            layout_input.delete_cell_rec(cell_index)
    
        return layout_new, cell_indices
    

    Test:

    layout = multi_box_layout2()
    clipped_layout, cell_indices = create_clipped_layout(layout, clip_layers, clip_boxes)
    test_layout(clipped_layout, layout)
    

    Please note, that the first test remains valid, I just left it out for brevity.

  • edited April 2023

    Hi @Peter123,

    That is not the problem of multi-clip. It's a problem of delete_cell_rec. This method will delete all cells of the sub-tree, even those which are used otherwise.

    Let me explain why that is an issue: the in-layout clip and the multi-clip have a nice feature, namely reusing cells as far as possible. If a cell fits entirely into the clip, the clip cell will simply instantiate this cell and not modify it. Hence the clip cell will reference cells from the original cell hierarchy. The beauty of that approach is that this is a zero-cost operation. The multi-clip will even reuse cells across multiple clips if possible.

    On the other hand, if you delete the clip cell recursively, this may also delete cells from your original hierarchy. The solution is to use prune_cell instead of delete_cell_rec which avoids that. prune_cell will only delete cells which are not used otherwise, hence not destroy your original hierarchy.

    Another solution is simple to save the layout before the clip and reload it again, but I guess you have your reasons for not doing that.

    And I see that a "clip_into" with a layer mapping object might be helpful.

    Kind regards,

    Matthias

  • edited April 2023

    Wow! It works like a charm when i substitute with prune_cell(cell,-1). Thanks a lot @Matthias for your aid and continuous KLayout support!

    You are absolutely right about the copying beforehand. I thought about this as well, but didnt like the idea to keep multiple instance of potentially very large layouts in memory and dealing with the additional overhead when writing to disk. Moreover, I consider functional programming using pure functions as very clean, hence I tried to do that. :)

    For anyone wondering what the final implementation looks like:

    def create_clipped_layout(layout_input: pya.Layout, layers: List[pya.LayerInfo],
                                              clips: List[pya.Box]) -> Tuple[pya.Layout, List[int]]:
        """
        create a clipped layout according to the given boxes / layers
        see: https://www.klayout.de/forum/discussion/2266/bug-inserting-a-layer-into-a-layout-restores-previously-clipped-shapes
        :param layout_input: the base layout to be clipped
        :param layers: the layers to be clipped
        :param clips: the boxes to be clipped
        :return: a clipped layout and the (top)cell indices of the clips
        """
        layout_new = pya.Layout()
        layout_new.dbu = layout_input.dbu
    
        old_cell_indices = layout_input.multi_clip(
            layout_input.top_cell().cell_index(), clips
        )
    
        cell_indices = []
        for index, cell_index in enumerate(old_cell_indices):
    
            new_cell = layout_new.create_cell(str(index))
    
            # prepare a target hierarchy
            cm = pya.CellMapping()
            cm.for_single_cell_full(new_cell, layout_input.cell(cell_index))
    
            # prepare the layer map
            lm = pya.LayerMapping()
            for layer in layers:
                lm.map(layout_input.layer(layer), layout_new.layer(layer))
    
            new_cell.copy_tree_shapes(layout_input.cell(cell_index), cm, lm)
            cell_indices.append(new_cell.cell_index())
    
        for cell_index in old_cell_indices:
            layout_input.prune_cell(cell_index, -1)
    
        return layout_new, cell_indices
    
Sign In or Register to comment.